@lloyal-labs/rig 1.2.1 → 1.2.2

package/dist/index.d.ts

@@ -1,18 +1,19 @@
 /**
  * Rig — research infrastructure for the lloyal agent pipeline
  *
- * The default export is RN-safe: no node:fs, no linkedom, no lloyal.node.
- * Platform-specific exports are available via sub-path imports:
+ * The default export is platform-agnostic. linkedom + @mozilla/readability
+ * are pure JS and work in both Node.js and React Native (Hermes).
  *
- * - `@lloyal-labs/rig/node` — Node.js-specific: createReranker, WebSource,
- *   CorpusSource, loadResources, chunkResources, FetchPageTool
+ * Node-specific exports (createReranker, WebSource, CorpusSource,
+ * loadResources, chunkResources) require node:fs and are available
+ * via `@lloyal-labs/rig/node`.
  *
  * @packageDocumentation
  * @category Rig
  */
-export { createTools, reportTool, ResearchTool, WebSearchTool, TavilyProvider, WebResearchTool, PlanTool, } from './tools';
+export { createTools, reportTool, ResearchTool, WebSearchTool, TavilyProvider, FetchPageTool, WebResearchTool, PlanTool, } from './tools';
 export type { ResearchToolOpts, WebResearchToolOpts, PlanToolOpts, PlanResult, PlanQuestion, SearchProvider, SearchResult, Reranker, ScoredChunk, ScoredResult, } from './tools';
-export { chunkFetchedPages } from './sources/chunking';
+export { chunkFetchedPages, chunkHtml } from './sources/chunking';
 export type { FetchedPage } from './sources/chunking';
 export type { SourceContext } from './sources/types';
 export type { Resource, Chunk } from './resources/types';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,EACL,WAAW,EAAE,UAAU,EACvB,YAAY,EAAE,aAAa,EAAE,cAAc,EAC3C,eAAe,EAAE,QAAQ,GAC1B,MAAM,SAAS,CAAC;AACjB,YAAY,EACV,gBAAgB,EAAE,mBAAmB,EAAE,YAAY,EACnD,UAAU,EAAE,YAAY,EACxB,cAAc,EAAE,YAAY,EAC5B,QAAQ,EAAE,WAAW,EAAE,YAAY,GACpC,MAAM,SAAS,CAAC;AAGjB,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGtD,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGrD,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,EACL,WAAW,EAAE,UAAU,EACvB,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,aAAa,EAC1D,eAAe,EAAE,QAAQ,GAC1B,MAAM,SAAS,CAAC;AACjB,YAAY,EACV,gBAAgB,EAAE,mBAAmB,EAAE,YAAY,EACnD,UAAU,EAAE,YAAY,EACxB,cAAc,EAAE,YAAY,EAC5B,QAAQ,EAAE,WAAW,EAAE,YAAY,GACpC,MAAM,SAAS,CAAC;AAGjB,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAClE,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGtD,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGrD,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC"}

package/dist/index.js

@@ -2,27 +2,30 @@
 /**
  * Rig — research infrastructure for the lloyal agent pipeline
  *
- * The default export is RN-safe: no node:fs, no linkedom, no lloyal.node.
- * Platform-specific exports are available via sub-path imports:
+ * The default export is platform-agnostic. linkedom + @mozilla/readability
+ * are pure JS and work in both Node.js and React Native (Hermes).
  *
- * - `@lloyal-labs/rig/node` — Node.js-specific: createReranker, WebSource,
- *   CorpusSource, loadResources, chunkResources, FetchPageTool
+ * Node-specific exports (createReranker, WebSource, CorpusSource,
+ * loadResources, chunkResources) require node:fs and are available
+ * via `@lloyal-labs/rig/node`.
  *
  * @packageDocumentation
  * @category Rig
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.chunkFetchedPages = exports.PlanTool = exports.WebResearchTool = exports.TavilyProvider = exports.WebSearchTool = exports.ResearchTool = exports.reportTool = exports.createTools = void 0;
-// Tools (all pure TS + Effection — RN-safe)
+exports.chunkHtml = exports.chunkFetchedPages = exports.PlanTool = exports.WebResearchTool = exports.FetchPageTool = exports.TavilyProvider = exports.WebSearchTool = exports.ResearchTool = exports.reportTool = exports.createTools = void 0;
+// Tools (pure TS + Effection + linkedom — platform-agnostic)
 var tools_1 = require("./tools");
 Object.defineProperty(exports, "createTools", { enumerable: true, get: function () { return tools_1.createTools; } });
 Object.defineProperty(exports, "reportTool", { enumerable: true, get: function () { return tools_1.reportTool; } });
 Object.defineProperty(exports, "ResearchTool", { enumerable: true, get: function () { return tools_1.ResearchTool; } });
 Object.defineProperty(exports, "WebSearchTool", { enumerable: true, get: function () { return tools_1.WebSearchTool; } });
 Object.defineProperty(exports, "TavilyProvider", { enumerable: true, get: function () { return tools_1.TavilyProvider; } });
+Object.defineProperty(exports, "FetchPageTool", { enumerable: true, get: function () { return tools_1.FetchPageTool; } });
 Object.defineProperty(exports, "WebResearchTool", { enumerable: true, get: function () { return tools_1.WebResearchTool; } });
 Object.defineProperty(exports, "PlanTool", { enumerable: true, get: function () { return tools_1.PlanTool; } });
-// Chunking (pure TS — RN-safe)
+// Chunking (platform-agnostic — linkedom is pure JS)
 var chunking_1 = require("./sources/chunking");
 Object.defineProperty(exports, "chunkFetchedPages", { enumerable: true, get: function () { return chunking_1.chunkFetchedPages; } });
+Object.defineProperty(exports, "chunkHtml", { enumerable: true, get: function () { return chunking_1.chunkHtml; } });
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;GAWG;;;AAEH,4CAA4C;AAC5C,iCAIiB;AAHf,oGAAA,WAAW,OAAA;AAAE,mGAAA,UAAU,OAAA;AACvB,qGAAA,YAAY,OAAA;AAAE,sGAAA,aAAa,OAAA;AAAE,uGAAA,cAAc,OAAA;AAC3C,wGAAA,eAAe,OAAA;AAAE,iGAAA,QAAQ,OAAA;AAS3B,+BAA+B;AAC/B,+CAAuD;AAA9C,6GAAA,iBAAiB,OAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;GAYG;;;AAEH,6DAA6D;AAC7D,iCAIiB;AAHf,oGAAA,WAAW,OAAA;AAAE,mGAAA,UAAU,OAAA;AACvB,qGAAA,YAAY,OAAA;AAAE,sGAAA,aAAa,OAAA;AAAE,uGAAA,cAAc,OAAA;AAAE,sGAAA,aAAa,OAAA;AAC1D,wGAAA,eAAe,OAAA;AAAE,iGAAA,QAAQ,OAAA;AAS3B,qDAAqD;AACrD,+CAAkE;AAAzD,6GAAA,iBAAiB,OAAA;AAAE,qGAAA,SAAS,OAAA"}

package/dist/node.d.ts

@@ -1,9 +1,8 @@
 /**
  * Node.js-specific exports for @lloyal-labs/rig
  *
- * These require node:fs, linkedom, @mozilla/readability, and/or
- * @lloyal-labs/lloyal.node. Import from `@lloyal-labs/rig/node`
- * only in Node.js environments.
+ * These require node:fs and/or @lloyal-labs/lloyal.node.
+ * Import from `@lloyal-labs/rig/node` only in Node.js environments.
  *
  * @packageDocumentation
  * @category Rig
@@ -12,6 +11,5 @@ export * from './index';
 export { createReranker } from './reranker';
 export { WebSource } from './sources/web';
 export { CorpusSource } from './sources/corpus';
-export { FetchPageTool } from './tools/fetch-page';
 export { loadResources, chunkResources } from './resources';
 //# sourceMappingURL=node.d.ts.map

package/dist/node.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"node.d.ts","sourceRoot":"","sources":["../src/node.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,cAAc,SAAS,CAAC;AAGxB,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAG5C,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAGhD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGnD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC"}
+{"version":3,"file":"node.d.ts","sourceRoot":"","sources":["../src/node.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,cAAc,SAAS,CAAC;AAGxB,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAG5C,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAGhD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC"}

package/dist/node.js

@@ -2,9 +2,8 @@
 /**
  * Node.js-specific exports for @lloyal-labs/rig
  *
- * These require node:fs, linkedom, @mozilla/readability, and/or
- * @lloyal-labs/lloyal.node. Import from `@lloyal-labs/rig/node`
- * only in Node.js environments.
+ * These require node:fs and/or @lloyal-labs/lloyal.node.
+ * Import from `@lloyal-labs/rig/node` only in Node.js environments.
  *
  * @packageDocumentation
  * @category Rig
@@ -24,20 +23,17 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.chunkResources = exports.loadResources = exports.FetchPageTool = exports.CorpusSource = exports.WebSource = exports.createReranker = void 0;
-// Re-export everything from the safe barrel
+exports.chunkResources = exports.loadResources = exports.CorpusSource = exports.WebSource = exports.createReranker = void 0;
+// Re-export everything from the platform-agnostic barrel
 __exportStar(require("./index"), exports);
 // Node-only: Reranker factory (requires @lloyal-labs/lloyal.node)
 var reranker_1 = require("./reranker");
 Object.defineProperty(exports, "createReranker", { enumerable: true, get: function () { return reranker_1.createReranker; } });
-// Node-only: Sources (require node:fs + linkedom)
+// Node-only: Sources (require node:fs)
 var web_1 = require("./sources/web");
 Object.defineProperty(exports, "WebSource", { enumerable: true, get: function () { return web_1.WebSource; } });
 var corpus_1 = require("./sources/corpus");
 Object.defineProperty(exports, "CorpusSource", { enumerable: true, get: function () { return corpus_1.CorpusSource; } });
-// Node-only: FetchPageTool (requires linkedom + @mozilla/readability)
-var fetch_page_1 = require("./tools/fetch-page");
-Object.defineProperty(exports, "FetchPageTool", { enumerable: true, get: function () { return fetch_page_1.FetchPageTool; } });
 // Node-only: Resource loading (requires node:fs)
 var resources_1 = require("./resources");
 Object.defineProperty(exports, "loadResources", { enumerable: true, get: function () { return resources_1.loadResources; } });

package/dist/node.js.map

@@ -1 +1 @@
-{"version":3,"file":"node.js","sourceRoot":"","sources":["../src/node.ts"],"names":[],"mappings":";AAAA;;;;;;;;;GASG;;;;;;;;;;;;;;;;;AAEH,4CAA4C;AAC5C,0CAAwB;AAExB,kEAAkE;AAClE,uCAA4C;AAAnC,0GAAA,cAAc,OAAA;AAEvB,kDAAkD;AAClD,qCAA0C;AAAjC,gGAAA,SAAS,OAAA;AAClB,2CAAgD;AAAvC,sGAAA,YAAY,OAAA;AAErB,sEAAsE;AACtE,iDAAmD;AAA1C,2GAAA,aAAa,OAAA;AAEtB,iDAAiD;AACjD,yCAA4D;AAAnD,0GAAA,aAAa,OAAA;AAAE,2GAAA,cAAc,OAAA"}
+{"version":3,"file":"node.js","sourceRoot":"","sources":["../src/node.ts"],"names":[],"mappings":";AAAA;;;;;;;;GAQG;;;;;;;;;;;;;;;;;AAEH,yDAAyD;AACzD,0CAAwB;AAExB,kEAAkE;AAClE,uCAA4C;AAAnC,0GAAA,cAAc,OAAA;AAEvB,uCAAuC;AACvC,qCAA0C;AAAjC,gGAAA,SAAS,OAAA;AAClB,2CAAgD;AAAvC,sGAAA,YAAY,OAAA;AAErB,iDAAiD;AACjD,yCAA4D;AAAnD,0GAAA,aAAa,OAAA;AAAE,2GAAA,cAAc,OAAA"}

package/dist/sources/chunking.d.ts

@@ -1,8 +1,11 @@
 /**
- * Pure-TS chunking utilities for web page content
+ * Chunking utilities for web page content
  *
- * Extracted from web.ts so they can be imported without pulling in
- * node:fs, linkedom, or @mozilla/readability.
+ * Two strategies:
+ * - {@link chunkHtml} — structural splitting on HTML headings/paragraphs
+ *   via linkedom. Used by FetchPageTool for per-tool reranking.
+ * - {@link chunkFetchedPages} — plain-text `\n\n` splitting for buffered
+ *   content. Used for post-research passage reranking.
  *
  * @packageDocumentation
  * @category Rig
@@ -38,4 +41,21 @@ export interface FetchedPage {
  * @category Rig
  */
 export declare function chunkFetchedPages(pages: FetchedPage[]): Chunk[];
+/**
+ * Split article HTML into heading-delimited section chunks via linkedom
+ *
+ * Same structural strategy as `parseMarkdown` (md4c) uses for corpus files:
+ * headings are section boundaries, content between headings is accumulated
+ * into a single chunk with the heading as metadata.
+ *
+ * Falls back to `<p>`-level chunks for pages without headings.
+ *
+ * @param html - Article HTML from Readability's `article.content`
+ * @param url - Page URL (used as chunk `resource`)
+ * @param title - Page title (used as default heading)
+ * @returns Array of section-level chunks
+ *
+ * @category Rig
+ */
+export declare function chunkHtml(html: string, url: string, title: string): Promise<Chunk[]>;
 //# sourceMappingURL=chunking.d.ts.map

package/dist/sources/chunking.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../../src/sources/chunking.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAEhD;;;;;;;;GAQG;AACH,MAAM,WAAW,WAAW;IAC1B,uCAAuC;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,uDAAuD;IACvD,KAAK,EAAE,MAAM,CAAC;IACd,kCAAkC;IAClC,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG,KAAK,EAAE,CAkC/D"}
+{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../../src/sources/chunking.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAEhD;;;;;;;;GAQG;AACH,MAAM,WAAW,WAAW;IAC1B,uCAAuC;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,uDAAuD;IACvD,KAAK,EAAE,MAAM,CAAC;IACd,kCAAkC;IAClC,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG,KAAK,EAAE,CAkC/D;AAKD;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAmD1F"}

package/dist/sources/chunking.js

@@ -1,15 +1,19 @@
 "use strict";
 /**
- * Pure-TS chunking utilities for web page content
+ * Chunking utilities for web page content
  *
- * Extracted from web.ts so they can be imported without pulling in
- * node:fs, linkedom, or @mozilla/readability.
+ * Two strategies:
+ * - {@link chunkHtml} — structural splitting on HTML headings/paragraphs
+ *   via linkedom. Used by FetchPageTool for per-tool reranking.
+ * - {@link chunkFetchedPages} — plain-text `\n\n` splitting for buffered
+ *   content. Used for post-research passage reranking.
  *
  * @packageDocumentation
  * @category Rig
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.chunkFetchedPages = chunkFetchedPages;
+exports.chunkHtml = chunkHtml;
 /**
  * Convert buffered web pages into {@link Chunk} instances for reranking
  *
@@ -55,4 +59,69 @@ function chunkFetchedPages(pages) {
     }
     return chunks;
 }
+const HEADING_TAGS = new Set(['H1', 'H2', 'H3', 'H4', 'H5', 'H6']);
+const TEXT_TAGS = new Set(['P', 'LI', 'BLOCKQUOTE', 'PRE', 'TD', 'TH', 'FIGCAPTION', 'DT', 'DD']);
+/**
+ * Split article HTML into heading-delimited section chunks via linkedom
+ *
+ * Same structural strategy as `parseMarkdown` (md4c) uses for corpus files:
+ * headings are section boundaries, content between headings is accumulated
+ * into a single chunk with the heading as metadata.
+ *
+ * Falls back to `<p>`-level chunks for pages without headings.
+ *
+ * @param html - Article HTML from Readability's `article.content`
+ * @param url - Page URL (used as chunk `resource`)
+ * @param title - Page title (used as default heading)
+ * @returns Array of section-level chunks
+ *
+ * @category Rig
+ */
+async function chunkHtml(html, url, title) {
+    const { parseHTML } = await Promise.resolve().then(() => require('linkedom'));
+    const { document } = parseHTML(html);
+    const chunks = [];
+    let currentHeading = title;
+    let currentText = '';
+    let chunkIndex = 0;
+    function flushSection() {
+        const text = currentText.trim();
+        if (text.length > 40) {
+            chunks.push({
+                resource: url,
+                heading: currentHeading || title || url,
+                text,
+                tokens: [],
+                startLine: chunkIndex + 1,
+                endLine: chunkIndex + 1,
+            });
+            chunkIndex++;
+        }
+        currentText = '';
+    }
+    // Walk all elements in the article DOM
+    const elements = document.querySelectorAll('*');
+    for (const el of elements) {
+        const tag = el.tagName;
+        if (HEADING_TAGS.has(tag)) {
+            // Close current section, start new one with this heading
+            flushSection();
+            currentHeading = el.textContent?.trim() || title;
+        }
+        else if (TEXT_TAGS.has(tag)) {
+            // Accumulate text content — skip if this element is nested inside
+            // another TEXT_TAG (avoid double-counting nested <li> etc.)
+            const parentTag = el.parentElement?.tagName;
+            if (parentTag && TEXT_TAGS.has(parentTag))
+                continue;
+            const text = el.textContent?.trim();
+            if (text) {
+                currentText += (currentText ? '\n\n' : '') + text;
+            }
+        }
+    }
+    // Flush final section
+    flushSection();
+    return chunks;
+}
 //# sourceMappingURL=chunking.js.map

package/dist/sources/chunking.js.map

@@ -1 +1 @@
-{"version":3,"file":"chunking.js","sourceRoot":"","sources":["../../src/sources/chunking.ts"],"names":[],"mappings":";AAAA;;;;;;;;GAQG;;AAkCH,8CAkCC;AA9CD;;;;;;;;;;;GAWG;AACH,SAAgB,iBAAiB,CAAC,KAAoB;IACpD,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI;aACzB,KAAK,CAAC,SAAS,CAAC;aAChB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;QAEhC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;gBACjC,MAAM,CAAC,IAAI,CAAC;oBACV,QAAQ,EAAE,IAAI,CAAC,GAAG;oBAClB,OAAO,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG;oBAC/B,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;oBACtB,MAAM,EAAE,EAAE;oBACV,SAAS,EAAE,CAAC;oBACZ,OAAO,EAAE,CAAC;iBACX,CAAC,CAAC;YACL,CAAC;YACD,SAAS;QACX,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,IAAI,CAAC,GAAG;gBAClB,OAAO,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG;gBAC/B,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC;gBACnB,MAAM,EAAE,EAAE;gBACV,SAAS,EAAE,CAAC,GAAG,CAAC;gBAChB,OAAO,EAAE,CAAC,GAAG,CAAC;aACf,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
+{"version":3,"file":"chunking.js","sourceRoot":"","sources":["../../src/sources/chunking.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;GAWG;;AAkCH,8CAkCC;AAqBD,8BAmDC;AAtHD;;;;;;;;;;;GAWG;AACH,SAAgB,iBAAiB,CAAC,KAAoB;IACpD,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI;aACzB,KAAK,CAAC,SAAS,CAAC;aAChB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;QAEhC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;gBACjC,MAAM,CAAC,IAAI,CAAC;oBACV,QAAQ,EAAE,IAAI,CAAC,GAAG;oBAClB,OAAO,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG;oBAC/B,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;oBACtB,MAAM,EAAE,EAAE;oBACV,SAAS,EAAE,CAAC;oBACZ,OAAO,EAAE,CAAC;iBACX,CAAC,CAAC;YACL,CAAC;YACD,SAAS;QACX,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,IAAI,CAAC,GAAG;gBAClB,OAAO,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG;gBAC/B,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC;gBACnB,MAAM,EAAE,EAAE;gBACV,SAAS,EAAE,CAAC,GAAG,CAAC;gBAChB,OAAO,EAAE,CAAC,GAAG,CAAC;aACf,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AACnE,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AAElG;;;;;;;;;;;;;;;GAeG;AACI,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,GAAW,EAAE,KAAa;IACtE,MAAM,EAAE,SAAS,EAAE,GAAG,2CAAa,UAAU,EAAC,CAAC;IAC/C,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAErC,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,cAAc,GAAG,KAAK,CAAC;IAC3B,IAAI,WAAW,GAAG,EAAE,CAAC;IACrB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,SAAS,YAAY;QACnB,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;QAChC,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,GAAG;gBACb,OAAO,EAAE,cAAc,IAAI,KAAK,IAAI,GAAG;gBACvC,IAAI;gBACJ,MAAM,EAAE,EAAE;gBACV,SAAS,EAAE,UAAU,GAAG,CAAC;gBACzB,OAAO,EAAE,UAAU,GAAG,CAAC;aACxB,CAAC,CAAC;YACH,UAAU,EAAE,CAAC;QACf,CAAC;QACD,WAAW,GAAG,EAAE,CAAC;IACnB,CAAC;IAED,uCAAuC;IACvC,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;IAChD,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC;QAEvB,IAAI,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,yDAAyD;YACzD,YAAY,EAAE,CAAC;YACf,cAAc,GAAG,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC;QACnD,CAAC;aAAM,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9B,kEAAkE;YAClE,4DAA4D;YAC5D,MAAM,SAAS,GAAG,EAAE,CAAC,aAAa,EAAE,OAAO,CAAC;YAC5C,IAAI,SAAS,IAAI,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC;gBAAE,SAAS;YAEpD,MAAM,IAAI,GAAG,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;YACpC,IAAI,IAAI,EAAE,CAAC;gBACT,WAAW,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC;YACpD,CAAC;QACH,CAAC;IACH,CAAC;IAED,sBAAsB;IACtB,YAAY,EAAE,CAAC;IAEf,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/sources/web.d.ts

@@ -1,18 +1,17 @@
 import type { Operation } from "effection";
 import { Source } from "@lloyal-labs/lloyal-agents";
-import { Tool } from "@lloyal-labs/lloyal-agents";
+import type { Tool } from "@lloyal-labs/lloyal-agents";
 import type { Chunk } from "../resources/types";
 import type { SourceContext } from "./types";
 import type { SearchProvider } from "../tools/types";
 export { chunkFetchedPages, type FetchedPage } from "./chunking";
 /**
- * Web-backed research source using search + fetch with scratchpad extraction
+ * Web-backed research source
  *
- * Wires up {@link BufferingWebSearch} and {@link BufferingFetchPage} for
- * grounding, and a self-referential {@link WebResearchTool} for spawning
- * parallel research sub-agents. Fetched page content is buffered in memory;
- * after research completes, {@link getChunks} converts the buffer into
- * {@link Chunk} instances via {@link chunkFetchedPages} for reranker scoring.
+ * Agents search the web via {@link WebSearchTool} (returns titles, snippets,
+ * URLs), then fetch promising pages via {@link FetchPageTool} (returns
+ * reranked relevant chunks). Fetched content is buffered for post-research
+ * passage reranking via {@link getChunks}.
  *
  * @category Rig
  */
@@ -33,12 +32,7 @@ export declare class WebSource extends Source<SourceContext, Chunk> {
     /** @inheritDoc */
     get groundingTools(): Tool[];
     /**
-     * Clear the page buffer and build the self-referential research toolkit
-     *
-     * Resets the internal {@link FetchedPage} buffer on every call so
-     * prior-run content does not leak into a new research pass. Constructs
-     * the {@link WebResearchTool} on first bind only (toolkit is stateless
-     * once built).
+     * Wire reranker to FetchPageTool and build the research toolkit
      *
      * @inheritDoc
      */

package/dist/sources/web.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"web.d.ts","sourceRoot":"","sources":["../../src/sources/web.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AACpD,OAAO,EAAE,IAAI,EAAyD,MAAM,4BAA4B,CAAC;AAIzG,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAQrD,OAAO,EAAE,iBAAiB,EAAE,KAAK,WAAW,EAAE,MAAM,YAAY,CAAC;AA+MjE;;;;;;;;;;GAUG;AACH,qBAAa,SAAU,SAAQ,MAAM,CAAC,aAAa,EAAE,KAAK,CAAC;IACzD,OAAO,CAAC,OAAO,CAAqB;IACpC,OAAO,CAAC,UAAU,CAAqB;IACvC,OAAO,CAAC,UAAU,CAAqB;IACvC,OAAO,CAAC,eAAe,CAAmC;IAC1D,OAAO,CAAC,aAAa,CAAgC;IAErD,kBAAkB;IAClB,QAAQ,CAAC,IAAI,SAAS;IAEtB;;OAEG;gBACS,QAAQ,EAAE,cAAc;IASpC,kBAAkB;IAClB,IAAI,YAAY,IAAI,IAAI,CAIvB;IAED,kBAAkB;IAClB,IAAI,cAAc,IAAI,IAAI,EAAE,CAA+C;IAE3E;;;;;;;;;OASG;IACF,IAAI,CAAC,GAAG,EAAE,aAAa,GAAG,SAAS,CAAC,IAAI,CAAC;IA2B1C,kBAAkB;IAClB,SAAS,IAAI,KAAK,EAAE;CAGrB"}
+{"version":3,"file":"web.d.ts","sourceRoot":"","sources":["../../src/sources/web.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,MAAM,EAAwB,MAAM,4BAA4B,CAAC;AAC1E,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,4BAA4B,CAAC;AACvD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAQrD,OAAO,EAAE,iBAAiB,EAAE,KAAK,WAAW,EAAE,MAAM,YAAY,CAAC;AAmDjE;;;;;;;;;GASG;AACH,qBAAa,SAAU,SAAQ,MAAM,CAAC,aAAa,EAAE,KAAK,CAAC;IACzD,OAAO,CAAC,OAAO,CAAqB;IACpC,OAAO,CAAC,UAAU,CAAqB;IACvC,OAAO,CAAC,UAAU,CAAgB;IAClC,OAAO,CAAC,eAAe,CAAmC;IAC1D,OAAO,CAAC,aAAa,CAAgC;IAErD,kBAAkB;IAClB,QAAQ,CAAC,IAAI,SAAS;IAEtB;;OAEG;gBACS,QAAQ,EAAE,cAAc;IAOpC,kBAAkB;IAClB,IAAI,YAAY,IAAI,IAAI,CAIvB;IAED,kBAAkB;IAClB,IAAI,cAAc,IAAI,IAAI,EAAE,CAE3B;IAED;;;;OAIG;IACF,IAAI,CAAC,GAAG,EAAE,aAAa,GAAG,SAAS,CAAC,IAAI,CAAC;IAoC1C,kBAAkB;IAClB,SAAS,IAAI,KAAK,EAAE;CAGrB"}

package/dist/sources/web.js

@@ -3,9 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.WebSource = exports.chunkFetchedPages = void 0;
 const fs = require("node:fs");
 const path = require("node:path");
-const effection_1 = require("effection");
 const lloyal_agents_1 = require("@lloyal-labs/lloyal-agents");
-const lloyal_agents_2 = require("@lloyal-labs/lloyal-agents");
 const web_search_1 = require("../tools/web-search");
 const fetch_page_1 = require("../tools/fetch-page");
 const web_research_1 = require("../tools/web-research");
@@ -25,185 +23,42 @@ function readTask(name) {
 }
 // ── BufferingFetchPage ───────────────────────────────────────────
 /**
- * Fetch-page wrapper that buffers full content and extracts a compact summary
+ * Thin wrapper over {@link FetchPageTool} that buffers fetched content
+ * for post-research reranking via {@link WebSource.getChunks}.
  *
- * Wraps {@link FetchPageTool} to intercept successful fetches. Full page
- * content is pushed into a shared {@link FetchedPage} buffer for
- * post-research reranking. An attention scratchpad (forked from
- * {@link ScratchpadParent}) then grammar-constrains a summary + links
- * extraction, returning the compact result to the calling agent instead
- * of the full page text. Falls back to the full result if extraction
- * fails or no scratchpad parent is available.
+ * No scratchpad extraction. No content transformation. Just buffers
+ * the raw text alongside returning the reranked result to the agent.
  *
  * @category Rig
  */
-class BufferingFetchPage extends lloyal_agents_2.Tool {
-    name = "fetch_page";
-    description = "Fetch a web page and extract its article content. Returns a summary and any links worth following. Use to read search results or follow links discovered in pages.";
-    parameters = {
-        type: "object",
-        properties: { url: { type: "string", description: "URL to fetch" } },
-        required: ["url"],
-    };
-    _inner;
+class BufferingFetchPage extends fetch_page_1.FetchPageTool {
     _buffer;
-    _extractTask;
-    constructor(buffer, extractTask, maxChars) {
-        super();
-        this._inner = new fetch_page_1.FetchPageTool(maxChars);
+    constructor(buffer, maxChars) {
+        super(maxChars);
         this._buffer = buffer;
-        this._extractTask = extractTask;
     }
     *execute(args) {
-        const result = yield* this._inner.execute(args);
+        const result = yield* super.execute(args);
         const r = result;
         if (typeof r?.content === "string" &&
             r.content !== "[Could not extract article content]") {
-            const content = r.content;
-            // Buffer full content for reranking
             this._buffer.push({
                 url: r.url || args.url,
                 title: r.title || "",
-                text: content,
+                text: r.content,
             });
-            // Attention scratchpad: fork from innermost active root, extract summary + links, prune
-            let parent;
-            try {
-                parent = yield* lloyal_agents_2.ScratchpadParent.expect();
-            }
-            catch { /* no parent — skip extraction */ }
-            if (!parent || parent.disposed)
-                return result;
-            const ctx = yield* lloyal_agents_2.Ctx.expect();
-            const schema = {
-                type: "object",
-                properties: {
-                    summary: { type: "string" },
-                    links: { type: "array", items: { type: "string" } },
-                },
-                required: ["summary", "links"],
-            };
-            const grammar = yield* (0, effection_1.call)(() => ctx.jsonSchemaToGrammar(JSON.stringify(schema)));
-            const extractPrompt = this._extractTask.user
-                .replace("{{url}}", args.url)
-                .replace("{{title}}", r.title || "")
-                .replace("{{content}}", content);
-            const messages = [
-                { role: "system", content: this._extractTask.system },
-                { role: "user", content: extractPrompt },
-            ];
-            const { prompt } = ctx.formatChatSync(JSON.stringify(messages), { enableThinking: false });
-            try {
-                const extracted = yield* (0, lloyal_agents_2.generate)({
-                    prompt,
-                    grammar,
-                    params: { temperature: 0.3 },
-                    parse: (o) => JSON.parse(o),
-                    parent,
-                });
-                return {
-                    url: r.url || args.url,
-                    title: r.title || "",
-                    summary: extracted.parsed?.summary || "",
-                    links: extracted.parsed?.links || [],
-                };
-            }
-            catch {
-                return result; // fallback to full result on extraction failure
-            }
         }
         return result;
     }
 }
-// ── BufferingWebSearch ────────────────────────────────────────────
-/**
- * Web-search wrapper that extracts a compact summary via attention scratchpad
- *
- * Wraps {@link WebSearchTool} and, when a {@link ScratchpadParent} is
- * available, forks a grammar-constrained generation to distill raw search
- * results into a list of promising URLs plus a brief summary. The compact
- * output reduces KV pressure on the calling agent. Falls back to raw
- * results if extraction fails or no scratchpad parent is available.
- *
- * @category Rig
- */
-class BufferingWebSearch extends lloyal_agents_2.Tool {
-    name = "web_search";
-    description = "Search the web. Returns the most relevant URLs and a summary. Use fetch_page to read full content of promising results.";
-    parameters = {
-        type: "object",
-        properties: { query: { type: "string", description: "Search query" } },
-        required: ["query"],
-    };
-    _inner;
-    _extractTask;
-    constructor(provider, extractTask) {
-        super();
-        this._inner = new web_search_1.WebSearchTool(provider);
-        this._extractTask = extractTask;
-    }
-    *execute(args) {
-        const results = yield* this._inner.execute(args);
-        // If error or not an array, return as-is (no scratchpad needed)
-        if (!Array.isArray(results) || results.length === 0)
-            return results;
-        // Scratchpad: fork from innermost active root, extract URLs + summary
-        let parent;
-        try {
-            parent = yield* lloyal_agents_2.ScratchpadParent.expect();
-        }
-        catch { /* no parent — return raw */ }
-        if (!parent || parent.disposed)
-            return results;
-        const ctx = yield* lloyal_agents_2.Ctx.expect();
-        const schema = {
-            type: "object",
-            properties: {
-                urls: { type: "array", items: { type: "string" }, description: "URLs worth fetching" },
-                summary: { type: "string", description: "Brief summary of what the search found" },
-            },
-            required: ["urls", "summary"],
-        };
-        const grammar = yield* (0, effection_1.call)(() => ctx.jsonSchemaToGrammar(JSON.stringify(schema)));
-        const resultsText = results
-            .map((r, i) => `${i + 1}. ${r.title}\n   ${r.url}\n   ${r.snippet}`)
-            .join("\n\n");
-        const extractPrompt = this._extractTask.user
-            .replace("{{query}}", args.query)
-            .replace("{{results}}", resultsText);
-        const messages = [
-            { role: "system", content: this._extractTask.system },
-            { role: "user", content: extractPrompt },
-        ];
-        const { prompt } = ctx.formatChatSync(JSON.stringify(messages), { enableThinking: false });
-        try {
-            const extracted = yield* (0, lloyal_agents_2.generate)({
-                prompt,
-                grammar,
-                params: { temperature: 0.3 },
-                parse: (o) => JSON.parse(o),
-                parent,
-            });
-            return {
-                urls: extracted.parsed?.urls || [],
-                summary: extracted.parsed?.summary || "",
-                resultCount: results.length,
-            };
-        }
-        catch {
-            return results; // fallback to raw results on extraction failure
-        }
-    }
-}
 // ── WebSource ────────────────────────────────────────────────────
 /**
- * Web-backed research source using search + fetch with scratchpad extraction
+ * Web-backed research source
  *
- * Wires up {@link BufferingWebSearch} and {@link BufferingFetchPage} for
- * grounding, and a self-referential {@link WebResearchTool} for spawning
- * parallel research sub-agents. Fetched page content is buffered in memory;
- * after research completes, {@link getChunks} converts the buffer into
- * {@link Chunk} instances via {@link chunkFetchedPages} for reranker scoring.
+ * Agents search the web via {@link WebSearchTool} (returns titles, snippets,
+ * URLs), then fetch promising pages via {@link FetchPageTool} (returns
+ * reranked relevant chunks). Fetched content is buffered for post-research
+ * passage reranking via {@link getChunks}.
  *
  * @category Rig
  */
@@ -220,11 +75,9 @@ class WebSource extends lloyal_agents_1.Source {
      */
     constructor(provider) {
         super();
-        const extractTask = readTask("extract");
-        const searchExtractTask = readTask("search-extract");
         this._researchPrompt = readTask("web-research");
-        this._fetchPage = new BufferingFetchPage(this._buffer, extractTask);
-        this._webSearch = new BufferingWebSearch(provider, searchExtractTask);
+        this._fetchPage = new BufferingFetchPage(this._buffer);
+        this._webSearch = new web_search_1.WebSearchTool(provider);
     }
     /** @inheritDoc */
     get researchTool() {
@@ -233,22 +86,26 @@ class WebSource extends lloyal_agents_1.Source {
         return this._researchTool;
     }
     /** @inheritDoc */
-    get groundingTools() { return [this._webSearch, this._fetchPage]; }
+    get groundingTools() {
+        return [this._webSearch, this._fetchPage];
+    }
     /**
-     * Clear the page buffer and build the self-referential research toolkit
-     *
-     * Resets the internal {@link FetchedPage} buffer on every call so
-     * prior-run content does not leak into a new research pass. Constructs
-     * the {@link WebResearchTool} on first bind only (toolkit is stateless
-     * once built).
+     * Wire reranker to FetchPageTool and build the research toolkit
      *
      * @inheritDoc
      */
     *bind(ctx) {
         this._buffer.length = 0;
-        const tw = yield* lloyal_agents_2.Trace.expect();
-        tw.write({ traceId: tw.nextId(), parentTraceId: null, ts: performance.now(),
-            type: 'source:bind', sourceName: this.name });
+        // Wire reranker to FetchPageTool for chunk scoring
+        this._fetchPage.setReranker(ctx.reranker);
+        const tw = yield* lloyal_agents_1.Trace.expect();
+        tw.write({
+            traceId: tw.nextId(),
+            parentTraceId: null,
+            ts: performance.now(),
+            type: "source:bind",
+            sourceName: this.name,
+        });
         if (!this._researchTool) {
             const webResearch = new web_research_1.WebResearchTool({
                 name: "web_research",
@@ -258,7 +115,7 @@ class WebSource extends lloyal_agents_1.Source {
                 maxTurns: ctx.maxTurns,
                 trace: ctx.trace,
             });
-            const toolkit = (0, lloyal_agents_2.createToolkit)([
+            const toolkit = (0, lloyal_agents_1.createToolkit)([
                 this._webSearch,
                 this._fetchPage,
                 ctx.reportTool,

package/dist/sources/web.js.map

@@ -1 +1 @@
-{"version":3,"file":"web.js","sourceRoot":"","sources":["../../src/sources/web.ts"],"names":[],"mappings":";;;AAAA,8BAA8B;AAC9B,kCAAkC;AAClC,yCAAiC;AAEjC,8DAAoD;AACpD,8DAAyG;AAOzG,oDAAoD;AACpD,oDAAoD;AACpD,wDAAwD;AACxD,yCAA+C;AAG/C,wCAAwC;AACxC,uCAAiE;AAAxD,6GAAA,iBAAiB,OAAA;AAE1B,oEAAoE;AAEpE,SAAS,QAAQ,CAAC,IAAY;IAC5B,MAAM,GAAG,GAAG,EAAE;SACX,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,IAAI,KAAK,CAAC,EAAE,MAAM,CAAC;SAC3D,IAAI,EAAE,CAAC;IACV,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACnC,IAAI,GAAG,KAAK,CAAC,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IACjD,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;AAC/E,CAAC;AAED,oEAAoE;AAEpE;;;;;;;;;;;;GAYG;AACH,MAAM,kBAAmB,SAAQ,oBAAqB;IAC3C,IAAI,GAAG,YAAY,CAAC;IACpB,WAAW,GAClB,oKAAoK,CAAC;IAC9J,UAAU,GAAe;QAChC,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,EAAE;QACpE,QAAQ,EAAE,CAAC,KAAK,CAAC;KAClB,CAAC;IAEM,MAAM,CAAgB;IACtB,OAAO,CAAgB;IACvB,YAAY,CAAmC;IAEvD,YACE,MAAqB,EACrB,WAA6C,EAC7C,QAAiB;QAEjB,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,IAAI,0BAAa,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;IAClC,CAAC;IAED,CAAC,OAAO,CAAC,IAAqB;QAC5B,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,CAAC,GAAG,MAAiC,CAAC;QAC5C,IACE,OAAO,CAAC,EAAE,OAAO,KAAK,QAAQ;YAC9B,CAAC,CAAC,OAAO,KAAK,qCAAqC,EACnD,CAAC;YACD,MAAM,OAAO,GAAG,CAAC,CAAC,OAAiB,CAAC;YACpC,oCAAoC;YACpC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;gBAChB,GAAG,EAAG,CAAC,CAAC,GAAc,IAAI,IAAI,CAAC,GAAG;gBAClC,KAAK,EAAG,CAAC,CAAC,KAAgB,IAAI,EAAE;gBAChC,IAAI,EAAE,OAAO;aACd,CAAC,CAAC;YAEH,wFAAwF;YACxF,IAAI,MAA0B,CAAC;YAC/B,IAAI,CAAC;gBAAC,MAAM,GAAG,KAAK,CAAC,CAAC,gCAAgB,CAAC,MAAM,EAAE,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,iCAAiC,CAAC,CAAC;YAC9F,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ;gBAAE,OAAO,MAAM,CAAC;YAC9C,MAAM,GAAG,GAAmB,KAAK,CAAC,CAAC,mBAAG,CAAC,MAAM,EAAE,CAAC;YAChD,MAAM,MAAM,GAAG;gBACb,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oBAC3B,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;iBACpD;gBACD,QAAQ,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC;aAC/B,CAAC;YACF,MAAM,OAAO,GAAW,KAAK,CAAC,CAAC,IAAA,gBAAI,EAAC,GAAG,EAAE,CACvC,GAAG,CAAC,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAChD,CAAC;YACF,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI;iBACzC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC;iBAC5B,OAAO,CAAC,WAAW,EAAG,CAAC,CAAC,KAAgB,IAAI,EAAE,CAAC;iBAC/C,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;YACnC,MAAM,QAAQ,GAAG;gBACf,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE;gBACrD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE;aACzC,CAAC;YACF,MAAM,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;YAE3F,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,IAAA,wBAAQ,EAC/B;oBACE,MAAM;oBACN,OAAO;oBACP,MAAM,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE;oBAC5B,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;oBAC3B,MAAM;iBACP,CACF,CAAC;gBACF,OAAO;oBACL,GAAG,EAAE,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG;oBACtB,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE;oBACpB,OAAO,EAAE,SAAS,CAAC,MAAM,EAAE,OAAO,IAAI,EAAE;oBACxC,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,KAAK,IAAI,EAAE;iBACrC,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,MAAM,CAAC,CAAC,gDAAgD;YACjE,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,qEAAqE;AAErE;;;;;;;;;;GAUG;AACH,MAAM,kBAAmB,SAAQ,oBAAuB;IAC7C,IAAI,GAAG,YAAY,CAAC;IACpB,WAAW,GAClB,yHAAyH,CAAC;IACnH,UAAU,GAAe;QAChC,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,EAAE;QACtE,QAAQ,EAAE,CAAC,OAAO,CAAC;KACpB,CAAC;IAEM,MAAM,CAAgB;IACtB,YAAY,CAAmC;IAEvD,YAAY,QAAwB,EAAE,WAA6C;QACjF,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,IAAI,0BAAa,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;IAClC,CAAC;IAED,CAAC,OAAO,CAAC,IAAuB;QAC9B,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAEjD,gEAAgE;QAChE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,OAAO,CAAC;QAEpE,sEAAsE;QACtE,IAAI,MAA0B,CAAC;QAC/B,IAAI,CAAC;YAAC,MAAM,GAAG,KAAK,CAAC,CAAC,gCAAgB,CAAC,MAAM,EAAE,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,4BAA4B,CAAC,CAAC;QACzF,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ;YAAE,OAAO,OAAO,CAAC;QAE/C,MAAM,GAAG,GAAmB,KAAK,CAAC,CAAC,mBAAG,CAAC,MAAM,EAAE,CAAC;QAChD,MAAM,MAAM,GAAG;YACb,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,IAAI,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,qBAAqB,EAAE;gBACtF,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,wCAAwC,EAAE;aACnF;YACD,QAAQ,EAAE,CAAC,MAAM,EAAE,SAAS,CAAC;SAC9B,CAAC;QACF,MAAM,OAAO,GAAW,KAAK,CAAC,CAAC,IAAA,gBAAI,EAAC,GAAG,EAAE,CACvC,GAAG,CAAC,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAChD,CAAC;QAEF,MAAM,WAAW,GAAI,OAAkE;aACpF,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC;aACnE,IAAI,CAAC,MAAM,CAAC,CAAC;QAChB,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI;aACzC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC;aAChC,OAAO,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;QACvC,MAAM,QAAQ,GAAG;YACf,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE;YACrD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE;SACzC,CAAC;QACF,MAAM,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;QAE3F,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,IAAA,wBAAQ,EAAsC;gBACrE,MAAM;gBACN,OAAO;gBACP,MAAM,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE;gBAC5B,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC3B,MAAM;aACP,CAAC,CAAC;YACH,OAAO;gBACL,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,IAAI,IAAI,EAAE;gBAClC,OAAO,EAAE,SAAS,CAAC,MAAM,EAAE,OAAO,IAAI,EAAE;gBACxC,WAAW,EAAE,OAAO,CAAC,MAAM;aAC5B,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,OAAO,CAAC,CAAC,gDAAgD;QAClE,CAAC;IACH,CAAC;CACF;AAED,oEAAoE;AAEpE;;;;;;;;;;GAUG;AACH,MAAa,SAAU,SAAQ,sBAA4B;IACjD,OAAO,GAAkB,EAAE,CAAC;IAC5B,UAAU,CAAqB;IAC/B,UAAU,CAAqB;IAC/B,eAAe,CAAmC;IAClD,aAAa,GAA2B,IAAI,CAAC;IAErD,kBAAkB;IACT,IAAI,GAAG,KAAK,CAAC;IAEtB;;OAEG;IACH,YAAY,QAAwB;QAClC,KAAK,EAAE,CAAC;QACR,MAAM,WAAW,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,iBAAiB,GAAG,QAAQ,CAAC,gBAAgB,CAAC,CAAC;QACrD,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAC,cAAc,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,IAAI,kBAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACpE,IAAI,CAAC,UAAU,GAAG,IAAI,kBAAkB,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;IACxE,CAAC;IAED,kBAAkB;IAClB,IAAI,YAAY;QACd,IAAI,CAAC,IAAI,CAAC,aAAa;YACrB,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED,kBAAkB;IAClB,IAAI,cAAc,KAAa,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAE3E;;;;;;;;;OASG;IACH,CAAC,IAAI,CAAC,GAAkB;QACtB,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,qBAAK,CAAC,MAAM,EAAE,CAAC;QACjC,EAAE,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,CAAC,GAAG,EAAE;YACzE,IAAI,EAAE,aAAa,EAAE,UAAU,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAEhD,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACxB,MAAM,WAAW,GAAG,IAAI,8BAAe,CAAC;gBACtC,IAAI,EAAE,cAAc;gBACpB,WAAW,EACT,2FAA2F;gBAC7F,YAAY,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;gBACzC,cAAc,EAAE,GAAG,CAAC,cAAc;gBAClC,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,KAAK,EAAE,GAAG,CAAC,KAAK;aACjB,CAAC,CAAC;YACH,MAAM,OAAO,GAAG,IAAA,6BAAa,EAAC;gBAC5B,IAAI,CAAC,UAAU;gBACf,IAAI,CAAC,UAAU;gBACf,GAAG,CAAC,UAAU;gBACd,WAAW;aACZ,CAAC,CAAC;YACH,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;YAChC,IAAI,CAAC,aAAa,GAAG,WAAW,CAAC;QACnC,CAAC;IACH,CAAC;IAED,kBAAkB;IAClB,SAAS;QACP,OAAO,IAAA,4BAAiB,EAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACzC,CAAC;CACF;AAzED,8BAyEC"}
+{"version":3,"file":"web.js","sourceRoot":"","sources":["../../src/sources/web.ts"],"names":[],"mappings":";;;AAAA,8BAA8B;AAC9B,kCAAkC;AAElC,8DAA0E;AAK1E,oDAAoD;AACpD,oDAAoD;AACpD,wDAAwD;AACxD,yCAA+C;AAG/C,wCAAwC;AACxC,uCAAiE;AAAxD,6GAAA,iBAAiB,OAAA;AAE1B,oEAAoE;AAEpE,SAAS,QAAQ,CAAC,IAAY;IAC5B,MAAM,GAAG,GAAG,EAAE;SACX,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,IAAI,KAAK,CAAC,EAAE,MAAM,CAAC;SAC3D,IAAI,EAAE,CAAC;IACV,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACnC,IAAI,GAAG,KAAK,CAAC,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IACjD,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;AAC/E,CAAC;AAED,oEAAoE;AAEpE;;;;;;;;GAQG;AACH,MAAM,kBAAmB,SAAQ,0BAAa;IACpC,OAAO,CAAgB;IAE/B,YAAY,MAAqB,EAAE,QAAiB;QAClD,KAAK,CAAC,QAAQ,CAAC,CAAC;QAChB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAED,CAAC,OAAO,CAAC,IAAqC;QAC5C,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,CAAC,GAAG,MAAiC,CAAC;QAC5C,IACE,OAAO,CAAC,EAAE,OAAO,KAAK,QAAQ;YAC9B,CAAC,CAAC,OAAO,KAAK,qCAAqC,EACnD,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;gBAChB,GAAG,EAAG,CAAC,CAAC,GAAc,IAAI,IAAI,CAAC,GAAG;gBAClC,KAAK,EAAG,CAAC,CAAC,KAAgB,IAAI,EAAE;gBAChC,IAAI,EAAE,CAAC,CAAC,OAAiB;aAC1B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,oEAAoE;AAEpE;;;;;;;;;GASG;AACH,MAAa,SAAU,SAAQ,sBAA4B;IACjD,OAAO,GAAkB,EAAE,CAAC;IAC5B,UAAU,CAAqB;IAC/B,UAAU,CAAgB;IAC1B,eAAe,CAAmC;IAClD,aAAa,GAA2B,IAAI,CAAC;IAErD,kBAAkB;IACT,IAAI,GAAG,KAAK,CAAC;IAEtB;;OAEG;IACH,YAAY,QAAwB;QAClC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAC,cAAc,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,IAAI,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvD,IAAI,CAAC,UAAU,GAAG,IAAI,0BAAa,CAAC,QAAQ,CAAC,CAAC;IAChD,CAAC;IAED,kBAAkB;IAClB,IAAI,YAAY;QACd,IAAI,CAAC,IAAI,CAAC,aAAa;YACrB,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED,kBAAkB;IAClB,IAAI,cAAc;QAChB,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAED;;;;OAIG;IACH,CAAC,IAAI,CAAC,GAAkB;QACtB,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QAExB,mDAAmD;QACnD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE1C,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,qBAAK,CAAC,MAAM,EAAE,CAAC;QACjC,EAAE,CAAC,KAAK,CAAC;YACP,OAAO,EAAE,EAAE,CAAC,MAAM,EAAE;YACpB,aAAa,EAAE,IAAI;YACnB,EAAE,EAAE,WAAW,CAAC,GAAG,EAAE;YACrB,IAAI,EAAE,aAAa;YACnB,UAAU,EAAE,IAAI,CAAC,IAAI;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACxB,MAAM,WAAW,GAAG,IAAI,8BAAe,CAAC;gBACtC,IAAI,EAAE,cAAc;gBACpB,WAAW,EACT,2FAA2F;gBAC7F,YAAY,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;gBACzC,cAAc,EAAE,GAAG,CAAC,cAAc;gBAClC,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,KAAK,EAAE,GAAG,CAAC,KAAK;aACjB,CAAC,CAAC;YACH,MAAM,OAAO,GAAG,IAAA,6BAAa,EAAC;gBAC5B,IAAI,CAAC,UAAU;gBACf,IAAI,CAAC,UAAU;gBACf,GAAG,CAAC,UAAU;gBACd,WAAW;aACZ,CAAC,CAAC;YACH,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;YAChC,IAAI,CAAC,aAAa,GAAG,WAAW,CAAC;QACnC,CAAC;IACH,CAAC;IAED,kBAAkB;IAClB,SAAS;QACP,OAAO,IAAA,4BAAiB,EAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACzC,CAAC;CACF;AA7ED,8BA6EC"}

package/dist/tools/fetch-page.d.ts

@@ -1,26 +1,41 @@
 import type { Operation } from 'effection';
 import { Tool } from '@lloyal-labs/lloyal-agents';
-import type { JsonSchema } from '@lloyal-labs/lloyal-agents';
+import type { JsonSchema, ToolContext } from '@lloyal-labs/lloyal-agents';
+import type { Reranker } from './types';
 /**
  * Fetch a web page and extract readable article content.
  *
  * Uses the Fetch API with a 10-second timeout, then extracts the
- * article body via linkedom + Readability. Content is truncated to
- * `maxChars` (default 6000). PDF URLs are rejected early since
- * binary content cannot be extracted as readable text.
+ * article body via linkedom + Readability.
+ *
+ * When a reranker is set (via {@link setReranker}) and the agent provides
+ * a `query` argument, the article HTML is structurally chunked on heading
+ * boundaries (same pattern as corpus `parseMarkdown`) and scored against
+ * the query. Only the top-K most relevant verbatim chunks are returned —
+ * reducing KV pressure without lossy summarization. The reranker runs on
+ * its own `llama_context`, consuming zero inference KV.
+ *
+ * Without a reranker or query, returns the full content truncated to
+ * `maxChars` (default 6000). Fully backward compatible.
  *
  * @category Rig
  */
 export declare class FetchPageTool extends Tool<{
     url: string;
+    query?: string;
 }> {
     readonly name = "fetch_page";
-    readonly description = "Fetch a web page and extract its article content. Returns readable text with title and excerpt. Use to read search results or follow links discovered in pages.";
+    readonly description = "Fetch a web page and extract its article content. Returns readable text with title and excerpt. Use to read search results or follow links discovered in pages. Pass a query to get only the most relevant sections.";
     readonly parameters: JsonSchema;
     private _maxChars;
-    constructor(maxChars?: number);
+    private _reranker;
+    private _topK;
+    constructor(maxChars?: number, topK?: number);
+    /** Inject reranker for chunk scoring. Call from Source.bind(). */
+    setReranker(reranker: Reranker): void;
     execute(args: {
         url: string;
-    }): Operation<unknown>;
+        query?: string;
+    }, context?: ToolContext): Operation<unknown>;
 }
 //# sourceMappingURL=fetch-page.d.ts.map

package/dist/tools/fetch-page.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fetch-page.d.ts","sourceRoot":"","sources":["../../src/tools/fetch-page.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,IAAI,EAAE,MAAM,4BAA4B,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAE7D;;;;;;;;;GASG;AACH,qBAAa,aAAc,SAAQ,IAAI,CAAC;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC;IACtD,QAAQ,CAAC,IAAI,gBAAgB;IAC7B,QAAQ,CAAC,WAAW,qKAAqK;IACzL,QAAQ,CAAC,UAAU,EAAE,UAAU,CAI7B;IAEF,OAAO,CAAC,SAAS,CAAS;gBAEd,QAAQ,SAAO;IAK1B,OAAO,CAAC,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,OAAO,CAAC;CA+CpD"}
+{"version":3,"file":"fetch-page.d.ts","sourceRoot":"","sources":["../../src/tools/fetch-page.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,IAAI,EAAS,MAAM,4BAA4B,CAAC;AACzD,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAE1E,OAAO,KAAK,EAAE,QAAQ,EAAe,MAAM,SAAS,CAAC;AAErD;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,aAAc,SAAQ,IAAI,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;IACtE,QAAQ,CAAC,IAAI,gBAAgB;IAC7B,QAAQ,CAAC,WAAW,0NAA0N;IAC9O,QAAQ,CAAC,UAAU,EAAE,UAAU,CAO7B;IAEF,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,SAAS,CAAyB;IAC1C,OAAO,CAAC,KAAK,CAAS;gBAEV,QAAQ,SAAO,EAAE,IAAI,SAAI;IAMrC,kEAAkE;IAClE,WAAW,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI;IAIpC,OAAO,CAAC,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC,OAAO,CAAC;CAgJ3F"}

package/dist/tools/fetch-page.js

@@ -3,40 +3,62 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.FetchPageTool = void 0;
 const effection_1 = require("effection");
 const lloyal_agents_1 = require("@lloyal-labs/lloyal-agents");
+const chunking_1 = require("../sources/chunking");
 /**
  * Fetch a web page and extract readable article content.
  *
  * Uses the Fetch API with a 10-second timeout, then extracts the
- * article body via linkedom + Readability. Content is truncated to
- * `maxChars` (default 6000). PDF URLs are rejected early since
- * binary content cannot be extracted as readable text.
+ * article body via linkedom + Readability.
+ *
+ * When a reranker is set (via {@link setReranker}) and the agent provides
+ * a `query` argument, the article HTML is structurally chunked on heading
+ * boundaries (same pattern as corpus `parseMarkdown`) and scored against
+ * the query. Only the top-K most relevant verbatim chunks are returned —
+ * reducing KV pressure without lossy summarization. The reranker runs on
+ * its own `llama_context`, consuming zero inference KV.
+ *
+ * Without a reranker or query, returns the full content truncated to
+ * `maxChars` (default 6000). Fully backward compatible.
  *
  * @category Rig
  */
 class FetchPageTool extends lloyal_agents_1.Tool {
     name = 'fetch_page';
-    description = 'Fetch a web page and extract its article content. Returns readable text with title and excerpt. Use to read search results or follow links discovered in pages.';
+    description = 'Fetch a web page and extract its article content. Returns readable text with title and excerpt. Use to read search results or follow links discovered in pages. Pass a query to get only the most relevant sections.';
     parameters = {
         type: 'object',
-        properties: { url: { type: 'string', description: 'URL to fetch' } },
+        properties: {
+            url: { type: 'string', description: 'URL to fetch' },
+            query: { type: 'string', description: 'What to look for in this page (optional — improves relevance of returned content)' },
+        },
         required: ['url'],
     };
     _maxChars;
-    constructor(maxChars = 6000) {
+    _reranker = null;
+    _topK;
+    constructor(maxChars = 6000, topK = 5) {
         super();
         this._maxChars = maxChars;
+        this._topK = topK;
     }
-    *execute(args) {
+    /** Inject reranker for chunk scoring. Call from Source.bind(). */
+    setReranker(reranker) {
+        this._reranker = reranker;
+    }
+    *execute(args, context) {
         const url = args.url?.trim();
         if (!url)
             return { error: 'url must not be empty' };
-        // Early reject PDF URLs — can't extract readable content from binary PDF
+        // Early reject PDF URLs
         const lowerUrl = url.toLowerCase();
         if (lowerUrl.endsWith('.pdf') || lowerUrl.includes('.pdf?') || lowerUrl.includes('.pdf#')) {
             return { error: 'PDF documents cannot be extracted. Try searching for an HTML version of this content.', url };
         }
         const maxChars = this._maxChars;
-        return yield* (0, effection_1.call)(async () => {
+        const reranker = this._reranker;
+        const topK = this._topK;
+        // Step 1: Fetch + readability (async)
+        const fetched = yield* (0, effection_1.call)(async () => {
             let res;
             try {
                 res = await fetch(url, {
@@ -60,12 +82,105 @@ class FetchPageTool extends lloyal_agents_1.Tool {
             const article = new Readability(document).parse();
             if (!article)
                 return { url, content: '[Could not extract article content]' };
-            let content = article.textContent ?? '';
-            if (content.length > maxChars) {
+            return {
+                url,
+                title: article.title ?? '',
+                content: article.textContent ?? '',
+                articleHtml: article.content ?? '',
+                excerpt: article.excerpt ?? '',
+            };
+        });
+        // Early return on error or no article
+        if ('error' in fetched)
+            return fetched;
+        if (!fetched.articleHtml) {
+            let content = fetched.content;
+            if (content.length > maxChars)
                 content = content.slice(0, maxChars) + '\n\n[truncated]';
+            return { url: fetched.url, title: fetched.title, content, excerpt: fetched.excerpt };
+        }
+        // Step 2: Reranker path — chunk HTML structurally, score, return top-K
+        if (reranker && args.query) {
+            const chunks = yield* (0, effection_1.call)(() => (0, chunking_1.chunkHtml)(fetched.articleHtml, url, fetched.title));
+            // Write chunks to trace for replay sufficiency
+            let tw;
+            try {
+                tw = yield* lloyal_agents_1.Trace.expect();
             }
-            return { url, title: article.title, content, excerpt: article.excerpt };
-        });
+            catch { /* no trace context */ }
+            const rerankT0 = performance.now();
+            if (tw) {
+                tw.write({
+                    traceId: tw.nextId(),
+                    parentTraceId: null,
+                    ts: rerankT0,
+                    type: 'rerank:start',
+                    query: args.query,
+                    chunkCount: chunks.length,
+                    tool: 'fetch_page',
+                    url,
+                    chunks: chunks.map(c => ({ heading: c.heading, textLength: c.text.length, startLine: c.startLine })),
+                });
+            }
+            if (chunks.length > 0) {
+                yield* (0, effection_1.call)(() => reranker.tokenizeChunks(chunks));
+                let scored = [];
+                yield* (0, effection_1.call)(async () => {
+                    for await (const batch of reranker.score(args.query, chunks)) {
+                        if (context?.onProgress)
+                            context.onProgress({ filled: batch.filled, total: batch.total });
+                        scored = batch.results;
+                    }
+                });
+                // Select top-K within token budget (tokens populated by tokenizeChunks)
+                const TOKEN_BUDGET = 2048;
+                const topChunks = [];
+                let tokenTotal = 0;
+                for (const sc of scored.slice(0, topK)) {
+                    const chunk = chunks.find(c => c.resource === sc.file && c.startLine === sc.startLine);
+                    if (!chunk?.text)
+                        continue;
+                    const chunkTokens = chunk.tokens.length || Math.ceil(chunk.text.length / 4);
+                    if (topChunks.length > 0 && tokenTotal + chunkTokens > TOKEN_BUDGET)
+                        break;
+                    topChunks.push({ text: chunk.text, heading: sc.heading, score: sc.score });
+                    tokenTotal += chunkTokens;
+                }
+                if (tw) {
+                    tw.write({
+                        traceId: tw.nextId(),
+                        parentTraceId: null,
+                        ts: performance.now(),
+                        type: 'rerank:end',
+                        topResults: topChunks.map(c => ({
+                            file: url,
+                            heading: c.heading,
+                            score: c.score,
+                            textPreview: c.text.slice(0, 200),
+                        })),
+                        selectedPassageCount: topChunks.length,
+                        totalChars: topChunks.reduce((sum, c) => sum + c.text.length, 0),
+                        durationMs: performance.now() - rerankT0,
+                        tool: 'fetch_page',
+                        url,
+                    });
+                }
+                if (topChunks.length > 0) {
+                    return {
+                        url,
+                        title: fetched.title,
+                        content: topChunks.map(c => c.text).join('\n\n---\n\n'),
+                        chunks: topChunks.length,
+                    };
+                }
+            }
+        }
+        // Fallback: return full content, truncated
+        let content = fetched.content;
+        if (content.length > maxChars) {
+            content = content.slice(0, maxChars) + '\n\n[truncated]';
+        }
+        return { url, title: fetched.title, content, excerpt: fetched.excerpt };
     }
 }
 exports.FetchPageTool = FetchPageTool;

package/dist/tools/fetch-page.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch-page.js","sourceRoot":"","sources":["../../src/tools/fetch-page.ts"],"names":[],"mappings":";;;AAAA,yCAAiC;AAEjC,8DAAkD;AAGlD;;;;;;;;;GASG;AACH,MAAa,aAAc,SAAQ,oBAAqB;IAC7C,IAAI,GAAG,YAAY,CAAC;IACpB,WAAW,GAAG,iKAAiK,CAAC;IAChL,UAAU,GAAe;QAChC,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,EAAE;QACpE,QAAQ,EAAE,CAAC,KAAK,CAAC;KAClB,CAAC;IAEM,SAAS,CAAS;IAE1B,YAAY,QAAQ,GAAG,IAAI;QACzB,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAED,CAAC,OAAO,CAAC,IAAqB;QAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,GAAG;YAAE,OAAO,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC;QAEpD,yEAAyE;QACzE,MAAM,QAAQ,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QACnC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1F,OAAO,EAAE,KAAK,EAAE,uFAAuF,EAAE,GAAG,EAAE,CAAC;QACjH,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,OAAO,KAAK,CAAC,CAAC,IAAA,gBAAI,EAAC,KAAK,IAAI,EAAE;YAC5B,IAAI,GAAa,CAAC;YAClB,IAAI,CAAC;gBACH,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;oBACrB,OAAO,EAAE,EAAE,YAAY,EAAE,6CAA6C,EAAE;oBACxE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;iBACpC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,KAAK,EAAE,iBAAkB,GAAa,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,CAAC;YACnE,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,CAAC;YAE3E,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC1D,IAAI,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBAC5C,OAAO,EAAE,KAAK,EAAE,uFAAuF,EAAE,GAAG,EAAE,CAAC;YACjH,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YAE9B,MAAM,EAAE,SAAS,EAAE,GAAG,2CAAa,UAAU,EAAC,CAAC;YAC/C,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;YAErC,MAAM,EAAE,WAAW,EAAE,GAAG,2CAAa,sBAAsB,EAAC,CAAC;YAC7D,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;YAElD,IAAI,CAAC,OAAO;gBAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,qCAAqC,EAAE,CAAC;YAE7E,IAAI,OAAO,GAAG,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC;YACxC,IAAI,OAAO,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;gBAC9B,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,GAAG,iBAAiB,CAAC;YAC3D,CAAC;YAED,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;QAC1E,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AA/DD,sCA+DC"}
+{"version":3,"file":"fetch-page.js","sourceRoot":"","sources":["../../src/tools/fetch-page.ts"],"names":[],"mappings":";;;AAAA,yCAAiC;AAEjC,8DAAyD;AAEzD,kDAAgD;AAGhD;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAa,aAAc,SAAQ,oBAAqC;IAC7D,IAAI,GAAG,YAAY,CAAC;IACpB,WAAW,GAAG,sNAAsN,CAAC;IACrO,UAAU,GAAe;QAChC,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE;YACpD,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,mFAAmF,EAAE;SAC5H;QACD,QAAQ,EAAE,CAAC,KAAK,CAAC;KAClB,CAAC;IAEM,SAAS,CAAS;IAClB,SAAS,GAAoB,IAAI,CAAC;IAClC,KAAK,CAAS;IAEtB,YAAY,QAAQ,GAAG,IAAI,EAAE,IAAI,GAAG,CAAC;QACnC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,kEAAkE;IAClE,WAAW,CAAC,QAAkB;QAC5B,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAED,CAAC,OAAO,CAAC,IAAqC,EAAE,OAAqB;QACnE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,GAAG;YAAE,OAAO,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC;QAEpD,wBAAwB;QACxB,MAAM,QAAQ,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QACnC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1F,OAAO,EAAE,KAAK,EAAE,uFAAuF,EAAE,GAAG,EAAE,CAAC;QACjH,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC;QAExB,sCAAsC;QACtC,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,IAAA,gBAAI,EAAC,KAAK,IAAI,EAAE;YACrC,IAAI,GAAa,CAAC;YAClB,IAAI,CAAC;gBACH,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;oBACrB,OAAO,EAAE,EAAE,YAAY,EAAE,6CAA6C,EAAE;oBACxE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;iBACpC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,KAAK,EAAE,iBAAkB,GAAa,CAAC,OAAO,EAAE,EAAE,GAAG,EAAW,CAAC;YAC5E,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,EAAE,GAAG,EAAW,CAAC;YAEpF,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC1D,IAAI,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBAC5C,OAAO,EAAE,KAAK,EAAE,uFAAuF,EAAE,GAAG,EAAW,CAAC;YAC1H,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YAE9B,MAAM,EAAE,SAAS,EAAE,GAAG,2CAAa,UAAU,EAAC,CAAC;YAC/C,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;YAErC,MAAM,EAAE,WAAW,EAAE,GAAG,2CAAa,sBAAsB,EAAC,CAAC;YAC7D,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;YAElD,IAAI,CAAC,OAAO;gBAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,qCAAqC,EAAW,CAAC;YAEtF,OAAO;gBACL,GAAG;gBACH,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,EAAE;gBAC1B,OAAO,EAAE,OAAO,CAAC,WAAW,IAAI,EAAE;gBAClC,WAAW,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE;gBAClC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE;aACtB,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,sCAAsC;QACtC,IAAI,OAAO,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;QACvC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YACzB,IAAI,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;YAC9B,IAAI,OAAO,CAAC,MAAM,GAAG,QAAQ;gBAAE,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,GAAG,iBAAiB,CAAC;YACxF,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;QACvF,CAAC;QAED,uEAAuE;QACvE,IAAI,QAAQ,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,IAAA,gBAAI,EAAC,GAAG,EAAE,CAAC,IAAA,oBAAS,EAAC,OAAO,CAAC,WAAW,EAAE,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;YAErF,+CAA+C;YAC/C,IAAI,EAAE,CAAC;YACP,IAAI,CAAC;gBAAC,EAAE,GAAG,KAAK,CAAC,CAAC,qBAAK,CAAC,MAAM,EAAE,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC;YACpE,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YACnC,IAAI,EAAE,EAAE,CAAC;gBACP,EAAE,CAAC,KAAK,CAAC;oBACP,OAAO,EAAE,EAAE,CAAC,MAAM,EAAE;oBACpB,aAAa,EAAE,IAAI;oBACnB,EAAE,EAAE,QAAQ;oBACZ,IAAI,EAAE,cAAc;oBACpB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,UAAU,EAAE,MAAM,CAAC,MAAM;oBACzB,IAAI,EAAE,YAAY;oBAClB,GAAG;oBACH,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;iBACrG,CAAC,CAAC;YACL,CAAC;YAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,KAAK,CAAC,CAAC,IAAA,gBAAI,EAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;gBAEnD,IAAI,MAAM,GAAkB,EAAE,CAAC;gBAC/B,KAAK,CAAC,CAAC,IAAA,gBAAI,EAAC,KAAK,IAAI,EAAE;oBACrB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,KAAM,EAAE,MAAM,CAAC,EAAE,CAAC;wBAC9D,IAAI,OAAO,EAAE,UAAU;4BAAE,OAAO,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;wBAC1F,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC;oBACzB,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,wEAAwE;gBACxE,MAAM,YAAY,GAAG,IAAI,CAAC;gBAC1B,MAAM,SAAS,GAA4D,EAAE,CAAC;gBAC9E,IAAI,UAAU,GAAG,CAAC,CAAC;gBACnB,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;oBACvC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,SAAS,KAAK,EAAE,CAAC,SAAS,CAAC,CAAC;oBACvF,IAAI,CAAC,KAAK,EAAE,IAAI;wBAAE,SAAS;oBAC3B,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBAC5E,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,GAAG,WAAW,GAAG,YAAY;wBAAE,MAAM;oBAC3E,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC;oBAC3E,UAAU,IAAI,WAAW,CAAC;gBAC5B,CAAC;gBAED,IAAI,EAAE,EAAE,CAAC;oBACP,EAAE,CAAC,KAAK,CAAC;wBACP,OAAO,EAAE,EAAE,CAAC,MAAM,EAAE;wBACpB,aAAa,EAAE,IAAI;wBACnB,EAAE,EAAE,WAAW,CAAC,GAAG,EAAE;wBACrB,IAAI,EAAE,YAAY;wBAClB,UAAU,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;4BAC9B,IAAI,EAAE,GAAG;4BACT,OAAO,EAAE,CAAC,CAAC,OAAO;4BAClB,KAAK,EAAE,CAAC,CAAC,KAAK;4BACd,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;yBAClC,CAAC,CAAC;wBACH,oBAAoB,EAAE,SAAS,CAAC,MAAM;wBACtC,UAAU,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;wBAChE,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,QAAQ;wBACxC,IAAI,EAAE,YAAY;wBAClB,GAAG;qBACJ,CAAC,CAAC;gBACL,CAAC;gBAED,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,OAAO;wBACL,GAAG;wBACH,KAAK,EAAE,OAAO,CAAC,KAAK;wBACpB,OAAO,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC;wBACvD,MAAM,EAAE,SAAS,CAAC,MAAM;qBACzB,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAED,2CAA2C;QAC3C,IAAI,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC9B,IAAI,OAAO,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YAC9B,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,GAAG,iBAAiB,CAAC;QAC3D,CAAC;QACD,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;IAC1E,CAAC;CACF;AA3KD,sCA2KC"}

package/dist/tools/index.d.ts

@@ -5,6 +5,7 @@ import { ReportTool } from './report';
 export { ResearchTool } from './research';
 export type { ResearchToolOpts } from './research';
 export { WebSearchTool, TavilyProvider } from './web-search';
+export { FetchPageTool } from './fetch-page';
 export type { SearchProvider, SearchResult, Reranker, ScoredChunk, ScoredResult } from './types';
 export { WebResearchTool } from './web-research';
 export type { WebResearchToolOpts } from './web-research';

package/dist/tools/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAIxC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,YAAY,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC7D,YAAY,EAAE,cAAc,EAAE,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACjG,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,YAAY,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAErE;;;;;;GAMG;AACH,eAAO,MAAM,UAAU,YAAmB,CAAC;AAE3C;;;;;;;;;;;GAWG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE;IAChC,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,QAAQ,EAAE,QAAQ,CAAC;CACpB,GAAG,OAAO,CAOV"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAIxC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,YAAY,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,YAAY,EAAE,cAAc,EAAE,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACjG,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,YAAY,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAErE;;;;;;GAMG;AACH,eAAO,MAAM,UAAU,YAAmB,CAAC;AAE3C;;;;;;;;;;;GAWG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE;IAChC,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,QAAQ,EAAE,QAAQ,CAAC;CACpB,GAAG,OAAO,CAOV"}

package/dist/tools/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.reportTool = exports.PlanTool = exports.WebResearchTool = exports.TavilyProvider = exports.WebSearchTool = exports.ResearchTool = void 0;
+exports.reportTool = exports.PlanTool = exports.WebResearchTool = exports.FetchPageTool = exports.TavilyProvider = exports.WebSearchTool = exports.ResearchTool = void 0;
 exports.createTools = createTools;
 const lloyal_agents_1 = require("@lloyal-labs/lloyal-agents");
 const search_1 = require("./search");
@@ -12,6 +12,8 @@ Object.defineProperty(exports, "ResearchTool", { enumerable: true, get: function
 var web_search_1 = require("./web-search");
 Object.defineProperty(exports, "WebSearchTool", { enumerable: true, get: function () { return web_search_1.WebSearchTool; } });
 Object.defineProperty(exports, "TavilyProvider", { enumerable: true, get: function () { return web_search_1.TavilyProvider; } });
+var fetch_page_1 = require("./fetch-page");
+Object.defineProperty(exports, "FetchPageTool", { enumerable: true, get: function () { return fetch_page_1.FetchPageTool; } });
 var web_research_1 = require("./web-research");
 Object.defineProperty(exports, "WebResearchTool", { enumerable: true, get: function () { return web_research_1.WebResearchTool; } });
 var plan_1 = require("./plan");

package/dist/tools/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":";;;AAuCA,kCAWC;AAlDD,8DAA2D;AAI3D,qCAAsC;AACtC,2CAA2C;AAC3C,iCAAkC;AAClC,qCAAsC;AAEtC,uCAA0C;AAAjC,wGAAA,YAAY,OAAA;AAErB,2CAA6D;AAApD,2GAAA,aAAa,OAAA;AAAE,4GAAA,cAAc,OAAA;AAEtC,+CAAiD;AAAxC,+GAAA,eAAe,OAAA;AAExB,+BAAkC;AAAzB,gGAAA,QAAQ,OAAA;AAGjB;;;;;;GAMG;AACU,QAAA,UAAU,GAAG,IAAI,mBAAU,EAAE,CAAC;AAE3C;;;;;;;;;;;GAWG;AACH,SAAgB,WAAW,CAAC,IAI3B;IACC,OAAO,IAAA,6BAAa,EAAC;QACnB,IAAI,mBAAU,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC;QAC1C,IAAI,wBAAY,CAAC,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,eAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;QAC5B,kBAAU;KACX,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":";;;AAwCA,kCAWC;AAnDD,8DAA2D;AAI3D,qCAAsC;AACtC,2CAA2C;AAC3C,iCAAkC;AAClC,qCAAsC;AAEtC,uCAA0C;AAAjC,wGAAA,YAAY,OAAA;AAErB,2CAA6D;AAApD,2GAAA,aAAa,OAAA;AAAE,4GAAA,cAAc,OAAA;AACtC,2CAA6C;AAApC,2GAAA,aAAa,OAAA;AAEtB,+CAAiD;AAAxC,+GAAA,eAAe,OAAA;AAExB,+BAAkC;AAAzB,gGAAA,QAAQ,OAAA;AAGjB;;;;;;GAMG;AACU,QAAA,UAAU,GAAG,IAAI,mBAAU,EAAE,CAAC;AAE3C;;;;;;;;;;;GAWG;AACH,SAAgB,WAAW,CAAC,IAI3B;IACC,OAAO,IAAA,6BAAa,EAAC;QACnB,IAAI,mBAAU,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC;QAC1C,IAAI,wBAAY,CAAC,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,eAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;QAC5B,kBAAU;KACX,CAAC,CAAC;AACL,CAAC"}

package/dist/tools/types.d.ts

@@ -66,6 +66,10 @@ export interface SearchResult {
     url: string;
     /** Excerpt or snippet from the page content */
     snippet: string;
+    /** Full page content — markdown when provider supports it, plain text otherwise */
+    rawContent?: string;
+    /** Provider-side relevance score (higher = more relevant) */
+    score?: number;
 }
 /**
  * Adapter interface for web search backends
@@ -80,5 +84,7 @@ export interface SearchResult {
 export interface SearchProvider {
     /** Execute a web search and return ranked results */
     search(query: string, maxResults: number): Promise<SearchResult[]>;
+    /** When true, rawContent on results is markdown with heading structure suitable for parseMarkdown chunking */
+    readonly returnsFullContentMarkdown: boolean;
 }
 //# sourceMappingURL=types.d.ts.map

package/dist/tools/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/tools/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAEhD;;;;;;;;;GASG;AACH,MAAM,WAAW,WAAW;IAC1B,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,2CAA2C;IAC3C,OAAO,EAAE,MAAM,CAAC;IAChB,+CAA+C;IAC/C,KAAK,EAAE,MAAM,CAAC;IACd,gDAAgD;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,8CAA8C;IAC9C,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,YAAY;IAC3B,6DAA6D;IAC7D,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,qCAAqC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,QAAQ;IACvB,kEAAkE;IAClE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;IACnE,uDAAuD;IACvD,cAAc,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C,iCAAiC;IACjC,OAAO,IAAI,IAAI,CAAC;CACjB;AAID;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,iBAAiB;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,eAAe;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,+CAA+C;IAC/C,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,cAAc;IAC7B,qDAAqD;IACrD,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;CACpE"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/tools/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAEhD;;;;;;;;;GASG;AACH,MAAM,WAAW,WAAW;IAC1B,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,2CAA2C;IAC3C,OAAO,EAAE,MAAM,CAAC;IAChB,+CAA+C;IAC/C,KAAK,EAAE,MAAM,CAAC;IACd,gDAAgD;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,8CAA8C;IAC9C,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,YAAY;IAC3B,6DAA6D;IAC7D,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,qCAAqC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,QAAQ;IACvB,kEAAkE;IAClE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;IACnE,uDAAuD;IACvD,cAAc,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C,iCAAiC;IACjC,OAAO,IAAI,IAAI,CAAC;CACjB;AAID;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,iBAAiB;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,eAAe;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,+CAA+C;IAC/C,OAAO,EAAE,MAAM,CAAC;IAChB,mFAAmF;IACnF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6DAA6D;IAC7D,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,cAAc;IAC7B,qDAAqD;IACrD,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IACnE,8GAA8G;IAC9G,QAAQ,CAAC,0BAA0B,EAAE,OAAO,CAAC;CAC9C"}

package/dist/tools/web-search.d.ts

@@ -13,8 +13,12 @@ export type { SearchProvider, SearchResult };
  * @category Rig
  */
 export declare class TavilyProvider implements SearchProvider {
+    readonly returnsFullContentMarkdown = false;
     private _apiKey;
-    constructor(apiKey?: string);
+    private _snippetMaxLength;
+    constructor(apiKey?: string, opts?: {
+        snippetMaxLength?: number;
+    });
     search(query: string, maxResults: number): Promise<SearchResult[]>;
 }
 /**

package/dist/tools/web-search.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"web-search.d.ts","sourceRoot":"","sources":["../../src/tools/web-search.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,IAAI,EAAE,MAAM,4BAA4B,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAE5D,YAAY,EAAE,cAAc,EAAE,YAAY,EAAE,CAAC;AAI7C;;;;;;;;GAQG;AACH,qBAAa,cAAe,YAAW,cAAc;IACnD,OAAO,CAAC,OAAO,CAAS;gBAEZ,MAAM,CAAC,EAAE,MAAM;IAIrB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAkBzE;AAID;;;;;;;;GAQG;AACH,qBAAa,aAAc,SAAQ,IAAI,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;IACxD,QAAQ,CAAC,IAAI,gBAAgB;IAC7B,QAAQ,CAAC,WAAW,gIAAgI;IACpJ,QAAQ,CAAC,UAAU,EAAE,UAAU,CAI7B;IAEF,OAAO,CAAC,SAAS,CAAiB;IAClC,OAAO,CAAC,KAAK,CAAS;gBAEV,QAAQ,EAAE,cAAc,EAAE,IAAI,SAAI;IAM7C,OAAO,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,OAAO,CAAC;CAYtD"}
+{"version":3,"file":"web-search.d.ts","sourceRoot":"","sources":["../../src/tools/web-search.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,IAAI,EAAE,MAAM,4BAA4B,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAE5D,YAAY,EAAE,cAAc,EAAE,YAAY,EAAE,CAAC;AAI7C;;;;;;;;GAQG;AACH,qBAAa,cAAe,YAAW,cAAc;IACnD,QAAQ,CAAC,0BAA0B,SAAS;IAC5C,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,iBAAiB,CAAS;gBAEtB,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,gBAAgB,CAAC,EAAE,MAAM,CAAA;KAAE;IAK3D,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAqBzE;AAID;;;;;;;;GAQG;AACH,qBAAa,aAAc,SAAQ,IAAI,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;IACxD,QAAQ,CAAC,IAAI,gBAAgB;IAC7B,QAAQ,CAAC,WAAW,gIAAgI;IACpJ,QAAQ,CAAC,UAAU,EAAE,UAAU,CAI7B;IAEF,OAAO,CAAC,SAAS,CAAiB;IAClC,OAAO,CAAC,KAAK,CAAS;gBAEV,QAAQ,EAAE,cAAc,EAAE,IAAI,SAAI;IAM7C,OAAO,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,OAAO,CAAC;CAYtD"}

package/dist/tools/web-search.js

@@ -14,9 +14,12 @@ const lloyal_agents_1 = require("@lloyal-labs/lloyal-agents");
  * @category Rig
  */
 class TavilyProvider {
+    returnsFullContentMarkdown = false;
     _apiKey;
-    constructor(apiKey) {
+    _snippetMaxLength;
+    constructor(apiKey, opts) {
         this._apiKey = apiKey || process.env.TAVILY_API_KEY || '';
+        this._snippetMaxLength = opts?.snippetMaxLength ?? 500;
     }
     async search(query, maxResults) {
         if (!this._apiKey)
@@ -32,10 +35,13 @@ class TavilyProvider {
         if (!res.ok)
             throw new Error(`Tavily ${res.status}: ${await res.text()}`);
         const data = await res.json();
+        const max = this._snippetMaxLength;
         return data.results.map(r => ({
             title: r.title,
             url: r.url,
-            snippet: r.content,
+            snippet: r.content.length > max
+                ? r.content.slice(0, max) + ' [\u2026]'
+                : r.content,
         }));
     }
 }

package/dist/tools/web-search.js.map

@@ -1 +1 @@
-{"version":3,"file":"web-search.js","sourceRoot":"","sources":["../../src/tools/web-search.ts"],"names":[],"mappings":";;;AAAA,yCAAiC;AAEjC,8DAAkD;AAMlD,2DAA2D;AAE3D;;;;;;;;GAQG;AACH,MAAa,cAAc;IACjB,OAAO,CAAS;IAExB,YAAY,MAAe;QACzB,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAAkB;QAC5C,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC7D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,+BAA+B,EAAE;YACvD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,eAAe,EAAE,UAAU,IAAI,CAAC,OAAO,EAAE;aAC1C;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC;SACzD,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,MAAM,KAAK,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1E,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAoE,CAAC;QAChG,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC5B,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,GAAG,EAAE,CAAC,CAAC,GAAG;YACV,OAAO,EAAE,CAAC,CAAC,OAAO;SACnB,CAAC,CAAC,CAAC;IACN,CAAC;CACF;AAzBD,wCAyBC;AAED,2DAA2D;AAE3D;;;;;;;;GAQG;AACH,MAAa,aAAc,SAAQ,oBAAuB;IAC/C,IAAI,GAAG,YAAY,CAAC;IACpB,WAAW,GAAG,4HAA4H,CAAC;IAC3I,UAAU,GAAe;QAChC,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,EAAE;QACtE,QAAQ,EAAE,CAAC,OAAO,CAAC;KACpB,CAAC;IAEM,SAAS,CAAiB;IAC1B,KAAK,CAAS;IAEtB,YAAY,QAAwB,EAAE,IAAI,GAAG,CAAC;QAC5C,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,CAAC,OAAO,CAAC,IAAuB;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,KAAK,EAAE,yBAAyB,EAAE,CAAC;QAExD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC;YACH,OAAO,KAAK,CAAC,CAAC,IAAA,gBAAI,EAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;QACzD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,kBAAmB,GAAa,CAAC,OAAO,EAAE,EAAE,CAAC;QAC/D,CAAC;IACH,CAAC;CACF;AA9BD,sCA8BC"}
+{"version":3,"file":"web-search.js","sourceRoot":"","sources":["../../src/tools/web-search.ts"],"names":[],"mappings":";;;AAAA,yCAAiC;AAEjC,8DAAkD;AAMlD,2DAA2D;AAE3D;;;;;;;;GAQG;AACH,MAAa,cAAc;IAChB,0BAA0B,GAAG,KAAK,CAAC;IACpC,OAAO,CAAS;IAChB,iBAAiB,CAAS;IAElC,YAAY,MAAe,EAAE,IAAoC;QAC/D,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC;QAC1D,IAAI,CAAC,iBAAiB,GAAG,IAAI,EAAE,gBAAgB,IAAI,GAAG,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAAkB;QAC5C,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC7D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,+BAA+B,EAAE;YACvD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,eAAe,EAAE,UAAU,IAAI,CAAC,OAAO,EAAE;aAC1C;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC;SACzD,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,MAAM,KAAK,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1E,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAoE,CAAC;QAChG,MAAM,GAAG,GAAG,IAAI,CAAC,iBAAiB,CAAC;QACnC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC5B,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,GAAG,EAAE,CAAC,CAAC,GAAG;YACV,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG;gBAC7B,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,WAAW;gBACvC,CAAC,CAAC,CAAC,CAAC,OAAO;SACd,CAAC,CAAC,CAAC;IACN,CAAC;CACF;AA/BD,wCA+BC;AAED,2DAA2D;AAE3D;;;;;;;;GAQG;AACH,MAAa,aAAc,SAAQ,oBAAuB;IAC/C,IAAI,GAAG,YAAY,CAAC;IACpB,WAAW,GAAG,4HAA4H,CAAC;IAC3I,UAAU,GAAe;QAChC,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,EAAE;QACtE,QAAQ,EAAE,CAAC,OAAO,CAAC;KACpB,CAAC;IAEM,SAAS,CAAiB;IAC1B,KAAK,CAAS;IAEtB,YAAY,QAAwB,EAAE,IAAI,GAAG,CAAC;QAC5C,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,CAAC,OAAO,CAAC,IAAuB;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,KAAK,EAAE,yBAAyB,EAAE,CAAC;QAExD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC;YACH,OAAO,KAAK,CAAC,CAAC,IAAA,gBAAI,EAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;QACzD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,kBAAmB,GAAa,CAAC,OAAO,EAAE,EAAE,CAAC;QAC/D,CAAC;IACH,CAAC;CACF;AA9BD,sCA8BC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lloyal-labs/rig",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "description": "Retrieval-Interleaved Generation for lloyal-agents",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",