@stackfactor/agent-utils 1.7.0 → 1.9.0

package/dist/esm/stackfactorKnowledge.d.ts

@@ -0,0 +1,182 @@
+/** What a STACKFACTOR reference to a knowledge base document carries (backend-core REFERENCE). */
+export declare const STACKFACTOR_REFERENCE: Readonly<{
+    KIND: "STACKFACTOR";
+    TARGET_ELEMENT_TYPE: "KNOWLEDGE_BASE_DOCUMENT";
+    ORIGIN: "KB_EXTRACTION";
+    VISIBILITY: "EDIT";
+}>;
+/** How many documents one search may be kept to (the backend's bound); more are split. */
+export declare const SEARCH_DOCUMENT_IDS_MAX = 500;
+/** How long a query may be (the backend's bound); longer ones are cut. */
+export declare const QUERY_MAX_LENGTH = 4000;
+/** How many passages one search asks for when the caller does not say. */
+export declare const DEFAULT_TOP_K = 8;
+/** A chunk as the platform's search answers it (client-api `RetrievedChunk`). */
+export interface KnowledgeChunk {
+    _id: string;
+    knowledgeBaseId?: string;
+    documentId: string;
+    documentName?: string | null;
+    sectionTitle?: string | null;
+    pageNumber?: number | null;
+    text?: string;
+    skills?: {
+        name: string | null;
+        microSkills?: string[];
+    }[];
+    score?: number | null;
+    [field: string]: unknown;
+}
+/** The platform's search, as client-api's `knowledgeBaseAgent.search` is called. */
+export type KnowledgeSearch = (knowledgeBaseId: string, query: string, options: {
+    topK?: number;
+    documentIds?: string[];
+}, token: string) => Promise<{
+    chunks: KnowledgeChunk[];
+}>;
+/** Where to search: a whole knowledge base, or only some of its documents. */
+export interface KnowledgeSource {
+    knowledgeBaseId: string;
+    /** Only these documents; omitted, the whole knowledge base. */
+    documentIds?: string[];
+}
+/** A passage handed to the model, tagged so it can cite it. */
+export interface GroundingPassage {
+    /** "k1", "k2"… in the order they rank. */
+    tag: string;
+    chunkId: string;
+    knowledgeBaseId: string;
+    documentId: string;
+    documentName: string | null;
+    sectionTitle: string | null;
+    pageNumber: number | null;
+    text: string;
+    /** The skills its passage teaches, as labelling named them. */
+    skills: {
+        name: string | null;
+        microSkills: string[];
+    }[];
+    score: number | null;
+}
+/** A knowledge base that could not be searched, and why. */
+export interface SkippedSource {
+    knowledgeBaseId: string;
+    /** The HTTP status, when the platform answered one. */
+    status: number | null;
+    /** True when the person may not read it (or it is gone): expected, not a fault. */
+    noAccess: boolean;
+    message: string;
+}
+/** A STACKFACTOR reference to the knowledge base document a passage came from. */
+export interface StackFactorReference {
+    kind: "STACKFACTOR";
+    title: string;
+    description: string | null;
+    target: {
+        elementType: "KNOWLEDGE_BASE_DOCUMENT";
+        elementId: string;
+        parentId: string;
+    };
+    knowledgeBase: {
+        knowledgeBaseId: string;
+        documentId: string;
+        chunkId: string;
+        sectionTitle: string | null;
+    };
+    origin: "KB_EXTRACTION";
+    visibility: "EDIT";
+}
+/**
+ * The knowledge bases, and the documents in them, that a list of references
+ * cites: its STACKFACTOR references to knowledge base documents, by their
+ * `knowledgeBase` ids, or by their `target` when only that is there. Any other
+ * reference is passed over. Each knowledge base once, its documents once each,
+ * in the order they first appear.
+ * @param references A skill's (or a microskill's) references
+ * @returns The sources to search
+ */
+export declare const sourcesFromReferences: (references: unknown) => KnowledgeSource[];
+/**
+ * The knowledge bases skill coverage found (client-api
+ * `knowledgeBase.getSkillCoverage`): every one the person can open that holds
+ * the skill, each searched whole.
+ * @param coverage The coverage answer, `{ knowledgeBases: [{ _id }] }`
+ * @returns The sources to search
+ */
+export declare const sourcesFromCoverage: (coverage: unknown) => KnowledgeSource[];
+/**
+ * Searches every source for every query, under the person's access, and hands
+ * back the passages found, best first, each once, tagged "k1", "k2"… for the
+ * model to cite. A source with documents is searched within them (in batches of
+ * SEARCH_DOCUMENT_IDS_MAX); one without, whole. A source the person may not
+ * read (401, 403, 404) is skipped: none of its passages are used and no more of
+ * its searches are made. A search that fails otherwise is reported and the
+ * passages its other searches found stand. Either way it is in `skipped`, once
+ * per knowledge base, and the passages of the others stand.
+ * @param options `sources`, `queries`, `search` (client-api
+ *   `knowledgeBaseAgent.search`), `token` (the run's AGENT token), `topK`
+ *   per search (default DEFAULT_TOP_K)
+ * @returns `{ passages, skipped }`
+ */
+export declare const retrievePassages: (options: {
+    sources: KnowledgeSource[];
+    queries: string[];
+    search: KnowledgeSearch;
+    token: string;
+    topK?: number;
+}) => Promise<{
+    passages: GroundingPassage[];
+    skipped: SkippedSource[];
+}>;
+/**
+ * The passages as a prompt shows them: tag, where each came from, and its text.
+ * @param passages From `retrievePassages`
+ * @returns `[{ tag, document, section, text }]`, for the model's input
+ */
+export declare const passagesForPrompt: (passages: GroundingPassage[]) => {
+    tag: string;
+    document: string | null;
+    section: string | null;
+    text: string;
+}[];
+/**
+ * STACKFACTOR references for passages: one per passage, in their order, with no
+ * cap. With `tags`, only the passages the model cited by tag (any tag it made
+ * up is passed over); without, every passage.
+ * @param passages From `retrievePassages`
+ * @param tags The tags the model cited ("k3", "K3" and " k3 " alike)
+ * @returns The references, ready for a skill's, a microskill's or a question's `references`
+ */
+export declare const referencesFor: (passages: GroundingPassage[], tags?: unknown[]) => StackFactorReference[];
+declare const _default: {
+    DEFAULT_TOP_K: number;
+    QUERY_MAX_LENGTH: number;
+    SEARCH_DOCUMENT_IDS_MAX: number;
+    STACKFACTOR_REFERENCE: Readonly<{
+        KIND: "STACKFACTOR";
+        TARGET_ELEMENT_TYPE: "KNOWLEDGE_BASE_DOCUMENT";
+        ORIGIN: "KB_EXTRACTION";
+        VISIBILITY: "EDIT";
+    }>;
+    passagesForPrompt: (passages: GroundingPassage[]) => {
+        tag: string;
+        document: string | null;
+        section: string | null;
+        text: string;
+    }[];
+    referencesFor: (passages: GroundingPassage[], tags?: unknown[]) => StackFactorReference[];
+    retrievePassages: (options: {
+        sources: KnowledgeSource[];
+        queries: string[];
+        search: KnowledgeSearch;
+        token: string;
+        topK?: number;
+    }) => Promise<{
+        passages: GroundingPassage[];
+        skipped: SkippedSource[];
+    }>;
+    sourcesFromCoverage: (coverage: unknown) => KnowledgeSource[];
+    sourcesFromReferences: (references: unknown) => KnowledgeSource[];
+};
+export default _default;
+//# sourceMappingURL=stackfactorKnowledge.d.ts.map

package/dist/esm/stackfactorKnowledge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stackfactorKnowledge.d.ts","sourceRoot":"","sources":["../../src/stackfactorKnowledge.ts"],"names":[],"mappings":"AA8BA,kGAAkG;AAClG,eAAO,MAAM,qBAAqB;;;;;EAKhC,CAAC;AAEH,0FAA0F;AAC1F,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAE3C,0EAA0E;AAC1E,eAAO,MAAM,gBAAgB,OAAO,CAAC;AAErC,0EAA0E;AAC1E,eAAO,MAAM,aAAa,IAAI,CAAC;AAQ/B,iFAAiF;AACjF,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;IAC3D,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,oFAAoF;AACpF,MAAM,MAAM,eAAe,GAAG,CAC5B,eAAe,EAAE,MAAM,EACvB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,EAClD,KAAK,EAAE,MAAM,KACV,OAAO,CAAC;IAAE,MAAM,EAAE,cAAc,EAAE,CAAA;CAAE,CAAC,CAAC;AAE3C,8EAA8E;AAC9E,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,MAAM,CAAC;IACxB,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,+DAA+D;AAC/D,MAAM,WAAW,gBAAgB;IAC/B,0CAA0C;IAC1C,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,+DAA+D;IAC/D,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,WAAW,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;IACzD,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,4DAA4D;AAC5D,MAAM,WAAW,aAAa;IAC5B,eAAe,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,mFAAmF;IACnF,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,kFAAkF;AAClF,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE;QAAE,WAAW,EAAE,yBAAyB,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IACxF,aAAa,EAAE;QACb,eAAe,EAAE,MAAM,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;KAC7B,CAAC;IACF,MAAM,EAAE,eAAe,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD;;;;;;;;GAQG;AACH,eAAO,MAAM,qBAAqB,GAAI,YAAY,OAAO,KAAG,eAAe,EAmB1E,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,mBAAmB,GAAI,UAAU,OAAO,KAAG,eAAe,EAWtE,CAAC;AA8BF;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,gBAAgB,GAAU,SAAS;IAC9C,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,eAAe,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,KAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAAC,OAAO,EAAE,aAAa,EAAE,CAAA;CAAE,CAgFrE,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,iBAAiB,GAC5B,UAAU,gBAAgB,EAAE,KAC3B;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,EAM5E,CAAC;AAEN;;;;;;;GAOG;AACH,eAAO,MAAM,aAAa,GACxB,UAAU,gBAAgB,EAAE,EAC5B,OAAO,OAAO,EAAE,KACf,oBAAoB,EAyBtB,CAAC;;;;;;;;;;;kCA7CU,gBAAgB,EAAE,KAC3B;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE;8BAiBvE,gBAAgB,EAAE,SACrB,OAAO,EAAE,KACf,oBAAoB,EAAE;gCAlHuB;QAC9C,OAAO,EAAE,eAAe,EAAE,CAAC;QAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,MAAM,EAAE,eAAe,CAAC;QACxB,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,KAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;QAAC,OAAO,EAAE,aAAa,EAAE,CAAA;KAAE,CAAC;oCA7DzB,OAAO,KAAG,eAAe,EAAE;wCA5BvB,OAAO,KAAG,eAAe,EAAE;;AAgO7E,wBAUE"}

package/dist/esm/stackfactorKnowledge.js

@@ -0,0 +1,266 @@
+//
+// StackFactor knowledge — grounding generated content in the knowledge bases
+//
+// A generation agent grounded in StackFactor (the Additional Information data
+// source "StackFactor", `dataSource: "RAG"`) writes from knowledge-base
+// passages instead of a web search, and cites every passage it used as a
+// STACKFACTOR reference. A later agent generating for the same skill (its
+// assessments, its learning content) reads those references and searches the
+// documents they cite. Designed with Candin, 2026-10-08:
+//
+//   - which knowledge bases: every one the person can open that holds the
+//     skill (skill coverage), never a fixed number of them;
+//   - which references: one per passage used, with no cap per document or per
+//     microskill;
+//   - access: the platform's search judges the person the run acts for, so a
+//     run reads only what that person may open. A knowledge base the person
+//     cannot read is skipped and reported, never an error: the agent
+//     generates without it.
+//
+// A reference points at the content and never copies it: the passage is
+// searched again, under the reader's own access, whenever it is used. It is
+// EDIT-only (`visibility: "EDIT"`): the platform strips it from what is
+// published, so a knowledge base's titles never reach a learner; agents read
+// it from the draft.
+//
+// The search itself is the caller's (`search`: client-api
+// `knowledgeBaseAgent.search`, 1.11 or later for `documentIds`), so this
+// module needs no particular client-api.
+//
+/** What a STACKFACTOR reference to a knowledge base document carries (backend-core REFERENCE). */
+export const STACKFACTOR_REFERENCE = Object.freeze({
+    KIND: "STACKFACTOR",
+    TARGET_ELEMENT_TYPE: "KNOWLEDGE_BASE_DOCUMENT",
+    ORIGIN: "KB_EXTRACTION",
+    VISIBILITY: "EDIT",
+});
+/** How many documents one search may be kept to (the backend's bound); more are split. */
+export const SEARCH_DOCUMENT_IDS_MAX = 500;
+/** How long a query may be (the backend's bound); longer ones are cut. */
+export const QUERY_MAX_LENGTH = 4000;
+/** How many passages one search asks for when the caller does not say. */
+export const DEFAULT_TOP_K = 8;
+/** How many searches run at once. */
+const CONCURRENCY = 4;
+/** Statuses that mean the person may not read the knowledge base, or it is gone. */
+const NO_ACCESS = new Set([401, 403, 404]);
+const text = (value) => (typeof value === "string" ? value.trim() : "");
+const id = (value) => typeof value === "string" ? value.trim() : value != null && typeof value === "object" ? String(value) : "";
+/**
+ * The knowledge bases, and the documents in them, that a list of references
+ * cites: its STACKFACTOR references to knowledge base documents, by their
+ * `knowledgeBase` ids, or by their `target` when only that is there. Any other
+ * reference is passed over. Each knowledge base once, its documents once each,
+ * in the order they first appear.
+ * @param references A skill's (or a microskill's) references
+ * @returns The sources to search
+ */
+export const sourcesFromReferences = (references) => {
+    const byBase = new Map();
+    for (const reference of Array.isArray(references) ? references : []) {
+        if (reference?.kind !== STACKFACTOR_REFERENCE.KIND)
+            continue;
+        const target = reference.target ?? {};
+        const isDocumentTarget = target.elementType === STACKFACTOR_REFERENCE.TARGET_ELEMENT_TYPE;
+        const knowledgeBaseId = id(reference.knowledgeBase?.knowledgeBaseId) || (isDocumentTarget ? id(target.parentId) : "");
+        const documentId = id(reference.knowledgeBase?.documentId) || (isDocumentTarget ? id(target.elementId) : "");
+        if (!knowledgeBaseId || !documentId)
+            continue;
+        const documents = byBase.get(knowledgeBaseId) ?? new Set();
+        documents.add(documentId);
+        byBase.set(knowledgeBaseId, documents);
+    }
+    return [...byBase.entries()].map(([knowledgeBaseId, documents]) => ({
+        knowledgeBaseId,
+        documentIds: [...documents],
+    }));
+};
+/**
+ * The knowledge bases skill coverage found (client-api
+ * `knowledgeBase.getSkillCoverage`): every one the person can open that holds
+ * the skill, each searched whole.
+ * @param coverage The coverage answer, `{ knowledgeBases: [{ _id }] }`
+ * @returns The sources to search
+ */
+export const sourcesFromCoverage = (coverage) => {
+    const bases = coverage?.knowledgeBases;
+    const seen = new Set();
+    const sources = [];
+    for (const base of Array.isArray(bases) ? bases : []) {
+        const knowledgeBaseId = id(base?._id);
+        if (!knowledgeBaseId || seen.has(knowledgeBaseId))
+            continue;
+        seen.add(knowledgeBaseId);
+        sources.push({ knowledgeBaseId });
+    }
+    return sources;
+};
+/** The documents of a source in batches the backend accepts; one batch of none for a whole knowledge base. */
+const batchesOf = (source) => {
+    if (!Array.isArray(source.documentIds))
+        return [undefined];
+    const batches = [];
+    for (let at = 0; at < source.documentIds.length; at += SEARCH_DOCUMENT_IDS_MAX) {
+        batches.push(source.documentIds.slice(at, at + SEARCH_DOCUMENT_IDS_MAX));
+    }
+    return batches;
+};
+/** Runs `worker` over `items`, at most `limit` at a time. */
+const inPool = async (items, limit, worker) => {
+    let next = 0;
+    const run = async () => {
+        while (next < items.length) {
+            const index = next++;
+            await worker(items[index]);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(limit, items.length) }, run));
+};
+/** The status of a failed platform call, from an axios error or one shaped like it. */
+const statusOf = (error) => {
+    const status = error?.response?.status ?? error?.status;
+    return typeof status === "number" ? status : null;
+};
+/**
+ * Searches every source for every query, under the person's access, and hands
+ * back the passages found, best first, each once, tagged "k1", "k2"… for the
+ * model to cite. A source with documents is searched within them (in batches of
+ * SEARCH_DOCUMENT_IDS_MAX); one without, whole. A source the person may not
+ * read (401, 403, 404) is skipped: none of its passages are used and no more of
+ * its searches are made. A search that fails otherwise is reported and the
+ * passages its other searches found stand. Either way it is in `skipped`, once
+ * per knowledge base, and the passages of the others stand.
+ * @param options `sources`, `queries`, `search` (client-api
+ *   `knowledgeBaseAgent.search`), `token` (the run's AGENT token), `topK`
+ *   per search (default DEFAULT_TOP_K)
+ * @returns `{ passages, skipped }`
+ */
+export const retrievePassages = async (options) => {
+    const queries = [
+        ...new Set((Array.isArray(options.queries) ? options.queries : [])
+            .map((query) => text(query).slice(0, QUERY_MAX_LENGTH))
+            .filter(Boolean)),
+    ];
+    const sources = Array.isArray(options.sources) ? options.sources : [];
+    const topK = options.topK ?? DEFAULT_TOP_K;
+    const searches = [];
+    for (const source of sources) {
+        for (const documentIds of batchesOf(source)) {
+            if (documentIds && documentIds.length === 0)
+                continue;
+            for (const query of queries) {
+                searches.push({ source, query, documentIds, order: searches.length });
+            }
+        }
+    }
+    const found = new Map();
+    const skipped = new Map();
+    await inPool(searches, CONCURRENCY, async ({ source, query, documentIds, order }) => {
+        if (skipped.get(source.knowledgeBaseId)?.noAccess)
+            return;
+        try {
+            const answer = await options.search(source.knowledgeBaseId, query, documentIds ? { topK, documentIds } : { topK }, options.token);
+            (Array.isArray(answer?.chunks) ? answer.chunks : []).forEach((chunk, rank) => {
+                const chunkId = id(chunk?._id);
+                if (!chunkId || !id(chunk?.documentId))
+                    return;
+                const known = found.get(chunkId);
+                const better = !known || (chunk.score ?? 0) > (known.chunk.score ?? 0);
+                // Ties keep the first search's rank, so the order is the searches' own.
+                found.set(chunkId, {
+                    chunk: better ? chunk : known.chunk,
+                    knowledgeBaseId: source.knowledgeBaseId,
+                    order: known ? Math.min(known.order, order * 1000 + rank) : order * 1000 + rank,
+                });
+            });
+        }
+        catch (error) {
+            const status = statusOf(error);
+            const noAccess = status !== null && NO_ACCESS.has(status);
+            // A refusal outranks a failure reported before it.
+            if (skipped.has(source.knowledgeBaseId) && !noAccess)
+                return;
+            skipped.set(source.knowledgeBaseId, {
+                knowledgeBaseId: source.knowledgeBaseId,
+                message: typeof error?.message === "string" ? error.message : String(error),
+                noAccess,
+                status,
+            });
+        }
+    });
+    const passages = [...found.entries()]
+        .filter(([, entry]) => !skipped.get(entry.knowledgeBaseId)?.noAccess)
+        .sort(([, a], [, b]) => (b.chunk.score ?? 0) - (a.chunk.score ?? 0) || a.order - b.order)
+        .map(([chunkId, { chunk, knowledgeBaseId }], index) => ({
+        chunkId,
+        documentId: id(chunk.documentId),
+        documentName: text(chunk.documentName) || null,
+        knowledgeBaseId: id(chunk.knowledgeBaseId) || knowledgeBaseId,
+        pageNumber: typeof chunk.pageNumber === "number" ? chunk.pageNumber : null,
+        score: typeof chunk.score === "number" ? chunk.score : null,
+        sectionTitle: text(chunk.sectionTitle) || null,
+        skills: (Array.isArray(chunk.skills) ? chunk.skills : []).map((skill) => ({
+            microSkills: (Array.isArray(skill?.microSkills) ? skill.microSkills : []).filter((name) => typeof name === "string"),
+            name: typeof skill?.name === "string" ? skill.name : null,
+        })),
+        tag: `k${index + 1}`,
+        text: typeof chunk.text === "string" ? chunk.text : "",
+    }));
+    return { passages, skipped: [...skipped.values()] };
+};
+/**
+ * The passages as a prompt shows them: tag, where each came from, and its text.
+ * @param passages From `retrievePassages`
+ * @returns `[{ tag, document, section, text }]`, for the model's input
+ */
+export const passagesForPrompt = (passages) => (Array.isArray(passages) ? passages : []).map((passage) => ({
+    document: passage.documentName,
+    section: passage.sectionTitle,
+    tag: passage.tag,
+    text: passage.text,
+}));
+/**
+ * STACKFACTOR references for passages: one per passage, in their order, with no
+ * cap. With `tags`, only the passages the model cited by tag (any tag it made
+ * up is passed over); without, every passage.
+ * @param passages From `retrievePassages`
+ * @param tags The tags the model cited ("k3", "K3" and " k3 " alike)
+ * @returns The references, ready for a skill's, a microskill's or a question's `references`
+ */
+export const referencesFor = (passages, tags) => {
+    const list = Array.isArray(passages) ? passages : [];
+    const cited = Array.isArray(tags)
+        ? new Set(tags.map((tag) => text(tag).toLowerCase()).filter(Boolean))
+        : null;
+    return list
+        .filter((passage) => !cited || cited.has(passage.tag.toLowerCase()))
+        .map((passage) => ({
+        description: passage.sectionTitle,
+        kind: STACKFACTOR_REFERENCE.KIND,
+        knowledgeBase: {
+            chunkId: passage.chunkId,
+            documentId: passage.documentId,
+            knowledgeBaseId: passage.knowledgeBaseId,
+            sectionTitle: passage.sectionTitle,
+        },
+        origin: STACKFACTOR_REFERENCE.ORIGIN,
+        target: {
+            elementId: passage.documentId,
+            elementType: STACKFACTOR_REFERENCE.TARGET_ELEMENT_TYPE,
+            parentId: passage.knowledgeBaseId,
+        },
+        title: passage.documentName || passage.sectionTitle || "Knowledge base document",
+        visibility: STACKFACTOR_REFERENCE.VISIBILITY,
+    }));
+};
+export default {
+    DEFAULT_TOP_K,
+    QUERY_MAX_LENGTH,
+    SEARCH_DOCUMENT_IDS_MAX,
+    STACKFACTOR_REFERENCE,
+    passagesForPrompt,
+    referencesFor,
+    retrievePassages,
+    sourcesFromCoverage,
+    sourcesFromReferences,
+};

package/dist/esm/tavily.d.ts

@@ -82,9 +82,40 @@ export declare const assertTavilyUsable: (config: any, options: TavilyConfig) =>
  * @returns LangChain tool instances to append to the agent's tool list
  */
 export declare const buildTavilyTools: (config: any, options: TavilyConfig, usageTracker?: UsageTracker | null) => any[];
+/**
+ * Which of these URLs resolve, judged by Tavily fetching them rather than by
+ * this process connecting to them.
+ *
+ * An agent that confirms a cited link by requesting it itself sends that
+ * request from inside StackFactor's network, through its fixed egress
+ * addresses, to wherever the link, a redirect or its DNS points — and cited
+ * links come from web pages anyone can publish, so that is a request to an
+ * address an outsider chose (server-side request forgery). Tavily fetches from
+ * its own infrastructure, so nothing here ever connects to such a host.
+ *
+ * Only `http:` and `https:` URLs are sent; any other is unreachable by
+ * definition. They go in batches of 20, Tavily's limit, at basic depth: 1
+ * credit per 5 URLs Tavily fetched, failed ones free. The credits are billed
+ * like every other Tavily call, but the URLs are not added to the tracker's
+ * sources: a link worth checking was found there in the first place.
+ *
+ * Tavily accounts for every URL it is given, listing the ones it could not
+ * fetch under `failed_results`. So a URL listed there is unreachable, and every
+ * other URL of a batch Tavily answered is reachable (the results may name a
+ * URL by its address after a redirect, which is why they are not matched). A
+ * batch whose call fails outright, or a config without `tavilyAPIKey`,
+ * confirms none of its URLs: a link that could not be checked is not trusted.
+ *
+ * @param urls - The URLs to check; a URL given twice is checked once
+ * @param config - The integration config; supplies `tavilyAPIKey`
+ * @param usageTracker - Optional accumulator the credits are counted into
+ * @returns The given URLs, exactly as passed, that resolve
+ */
+export declare const checkUrlsReachable: (urls: readonly string[], config: any, usageTracker?: UsageTracker | null) => Promise<Set<string>>;
 declare const _default: {
     assertTavilyUsable: (config: any, options: TavilyConfig) => void;
     buildTavilyTools: (config: any, options: TavilyConfig, usageTracker?: UsageTracker | null) => any[];
+    checkUrlsReachable: (urls: readonly string[], config: any, usageTracker?: UsageTracker | null) => Promise<Set<string>>;
     getTavilyOptions: (config: any) => TavilyConfig | null;
     recordTavilyUsage: (tracker: UsageTracker | null | undefined, _config: any, credits: number, sources: {
         url: string;

package/dist/esm/tavily.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tavily.d.ts","sourceRoot":"","sources":["../../src/tavily.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AA0BnD,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,KAAK,CAAC;AAEpE;;;;;;;;;GASG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,iEAAiE;IACjE,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,MAAM,GAAG,YAAY,CAAC;IAC3D,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;IACpC,2CAA2C;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oEAAoE;IACpE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qEAAqE;IACrE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4CAA4C;IAC5C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,6CAA6C;IAC7C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,uEAAuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,MAAM,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC;CAC9B,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,GAAI,QAAQ,GAAG,KAAG,YAAY,GAAG,IAI7D,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,kBAAkB,GAAI,QAAQ,GAAG,EAAE,SAAS,YAAY,KAAG,IAsBvE,CAAC;AAsLF;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,gBAAgB,GAC3B,QAAQ,GAAG,EACX,SAAS,YAAY,EACrB,eAAc,YAAY,GAAG,IAAW,KACvC,GAAG,EA2PL,CAAC;;iCA5dyC,GAAG,WAAW,YAAY,KAAG,IAAI;+BA8NlE,GAAG,WACF,YAAY,iBACP,YAAY,GAAG,IAAI,KAChC,GAAG,EAAE;+BAhPiC,GAAG,KAAG,YAAY,GAAG,IAAI;iCAsIvD,YAAY,GAAG,IAAI,GAAG,SAAS,WAC/B,GAAG,WACH,MAAM,WACN;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,KACzC,IAAI;;AAmWP,wBAKE"}
+{"version":3,"file":"tavily.d.ts","sourceRoot":"","sources":["../../src/tavily.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AA0BnD,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,KAAK,CAAC;AAEpE;;;;;;;;;GASG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,iEAAiE;IACjE,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,MAAM,GAAG,YAAY,CAAC;IAC3D,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;IACpC,2CAA2C;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oEAAoE;IACpE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qEAAqE;IACrE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4CAA4C;IAC5C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,6CAA6C;IAC7C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,uEAAuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,MAAM,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC;CAC9B,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,GAAI,QAAQ,GAAG,KAAG,YAAY,GAAG,IAI7D,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,kBAAkB,GAAI,QAAQ,GAAG,EAAE,SAAS,YAAY,KAAG,IAsBvE,CAAC;AAsLF;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,gBAAgB,GAC3B,QAAQ,GAAG,EACX,SAAS,YAAY,EACrB,eAAc,YAAY,GAAG,IAAW,KACvC,GAAG,EA2PL,CAAC;AAqBF;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,eAAO,MAAM,kBAAkB,GAC7B,MAAM,SAAS,MAAM,EAAE,EACvB,QAAQ,GAAG,EACX,eAAc,YAAY,GAAG,IAAW,KACvC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CA6DrB,CAAC;;iCA/kByC,GAAG,WAAW,YAAY,KAAG,IAAI;+BA8NlE,GAAG,WACF,YAAY,iBACP,YAAY,GAAG,IAAI,KAChC,GAAG,EAAE;+BA8SA,SAAS,MAAM,EAAE,UACf,GAAG,iBACG,YAAY,GAAG,IAAI,KAChC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;+BAjiBkB,GAAG,KAAG,YAAY,GAAG,IAAI;iCAsIvD,YAAY,GAAG,IAAI,GAAG,SAAS,WAC/B,GAAG,WACH,MAAM,WACN;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,KACzC,IAAI;;AAsdP,wBAME"}

package/dist/esm/tavily.js

@@ -386,9 +386,95 @@ export const buildTavilyTools = (config, options, usageTracker = null) => {
     logger.log(null, logger.levels.info, `Tavily tools enabled: ${tools.map((entry) => entry.name).join(", ")}`);
     return tools;
 };
+/** Tavily's own limit on URLs per `/extract` request. */
+const EXTRACT_MAX_URLS = 20;
+/** Seconds. A reachability check needs the page fetched, not rendered at length. */
+const REACHABILITY_TIMEOUT_SECONDS = 20;
+/** How a URL is matched against Tavily's answer: trimmed, lowercased, no trailing slash. */
+const urlMatchKey = (url) => url.trim().replace(/\/+$/, "").toLowerCase();
+/** True for an `http:` or `https:` URL; false for anything else, unparsable included. */
+const isWebUrl = (url) => {
+    try {
+        const { protocol } = new URL(url);
+        return protocol === "http:" || protocol === "https:";
+    }
+    catch {
+        return false;
+    }
+};
+/**
+ * Which of these URLs resolve, judged by Tavily fetching them rather than by
+ * this process connecting to them.
+ *
+ * An agent that confirms a cited link by requesting it itself sends that
+ * request from inside StackFactor's network, through its fixed egress
+ * addresses, to wherever the link, a redirect or its DNS points — and cited
+ * links come from web pages anyone can publish, so that is a request to an
+ * address an outsider chose (server-side request forgery). Tavily fetches from
+ * its own infrastructure, so nothing here ever connects to such a host.
+ *
+ * Only `http:` and `https:` URLs are sent; any other is unreachable by
+ * definition. They go in batches of 20, Tavily's limit, at basic depth: 1
+ * credit per 5 URLs Tavily fetched, failed ones free. The credits are billed
+ * like every other Tavily call, but the URLs are not added to the tracker's
+ * sources: a link worth checking was found there in the first place.
+ *
+ * Tavily accounts for every URL it is given, listing the ones it could not
+ * fetch under `failed_results`. So a URL listed there is unreachable, and every
+ * other URL of a batch Tavily answered is reachable (the results may name a
+ * URL by its address after a redirect, which is why they are not matched). A
+ * batch whose call fails outright, or a config without `tavilyAPIKey`,
+ * confirms none of its URLs: a link that could not be checked is not trusted.
+ *
+ * @param urls - The URLs to check; a URL given twice is checked once
+ * @param config - The integration config; supplies `tavilyAPIKey`
+ * @param usageTracker - Optional accumulator the credits are counted into
+ * @returns The given URLs, exactly as passed, that resolve
+ */
+export const checkUrlsReachable = async (urls, config, usageTracker = null) => {
+    const reachable = new Set();
+    const candidates = [
+        ...new Set(urls.filter((url) => typeof url === "string" && isWebUrl(url.trim()))),
+    ];
+    if (candidates.length === 0)
+        return reachable;
+    const apiKey = config?.tavilyAPIKey;
+    if (!apiKey) {
+        logger.log(null, logger.levels.warn, `Tavily reachability: no tavilyAPIKey, so none of ${candidates.length} URL(s) is confirmed`);
+        return reachable;
+    }
+    const batches = [];
+    for (let start = 0; start < candidates.length; start += EXTRACT_MAX_URLS) {
+        batches.push(candidates.slice(start, start + EXTRACT_MAX_URLS));
+    }
+    await Promise.all(batches.map(async (batch) => {
+        let data;
+        try {
+            data = await callTavily("extract", {
+                urls: batch.map((url) => url.trim()),
+                extract_depth: "basic",
+                format: "text",
+            }, apiKey, REACHABILITY_TIMEOUT_SECONDS);
+        }
+        catch (err) {
+            logger.log(null, logger.levels.warn, `Tavily reachability: the check of ${batch.length} URL(s) failed (${err?.message || String(err)}), so none of them is confirmed`);
+            return;
+        }
+        recordTavilyUsage(usageTracker, config, data?.usage?.credits || 0, []);
+        const failed = new Set((data?.failed_results || [])
+            .map((entry) => typeof entry?.url === "string" ? urlMatchKey(entry.url) : "")
+            .filter(Boolean));
+        for (const url of batch) {
+            if (!failed.has(urlMatchKey(url)))
+                reachable.add(url);
+        }
+    }));
+    return reachable;
+};
 export default {
     assertTavilyUsable,
     buildTavilyTools,
+    checkUrlsReachable,
     getTavilyOptions,
     recordTavilyUsage,
 };

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.7.0",
+  "version": "1.9.0",
   "description": "",
   "main": "dist/cjs/index.js",
   "module": "dist/esm/index.js",
@@ -21,7 +21,8 @@
     "release": "bash scripts/release.sh",
     "release:fanout": "bash scripts/release-fanout.sh",
     "ar-auth": "npm config set @stackfactor:registry https://us-central1-npm.pkg.dev/virtual-development-team/sf-devenv-npm-private/; TOKEN=$(gcloud auth print-access-token 2>/dev/null || curl -sfH 'Metadata-Flavor: Google' http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token | sed -n 's/.*\"access_token\":\"\\([^\"]*\\)\".*/\\1/p'); test -n \"$TOKEN\" || { echo 'ar-auth: no credential — run: gcloud auth login' >&2; exit 1; }; npm config set //us-central1-npm.pkg.dev/virtual-development-team/sf-devenv-npm-private/:_authToken \"$TOKEN\"; npm config set //us-central1-npm.pkg.dev/virtual-development-team/:_authToken \"$TOKEN\"; echo 'ar-auth: Artifact Registry scope + token configured'",
-    "reinstall": "npm run ar-auth && npm ci"
+    "reinstall": "npm run ar-auth && npm ci",
+    "test:coverage": "npm run build && node --experimental-test-coverage --test --test-reporter=tap test/*.test.js"
   },
   "repository": {
     "type": "git",
@@ -49,7 +50,7 @@
   "dependencies": {
     "@anthropic-ai/sdk": "^0.74.0",
     "@google/genai": "^1.46.0",
-    "@grpc/grpc-js": "^1.14.4",
+    "@grpc/grpc-js": "^1.14.5",
     "@grpc/proto-loader": "^0.7.15",
     "@langchain/anthropic": "^1.3.25",
     "@langchain/core": "^1.2.9",
@@ -58,7 +59,7 @@
     "@opentelemetry/api": "^1.9.0",
     "@stackfactor/client-api": "^1.6.0",
     "langchain": "^1.2.35",
-    "mongoose": "^9.3.1",
+    "mongoose": "^9.10.4",
     "openai": "^6.32.0",
     "winston": "^3.19.0",
     "zod": "^4.3.6"