@nhtio/adk 1.20260609.0 → 1.20260609.1

package/batteries/tools/web_retrieval.cjs

@@ -0,0 +1,206 @@
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+require("../../chunk-Ble4zEEl.js");
+const require_tool_registry = require("../../tool_registry-CKJPze3j.js");
+require("../../guards.cjs");
+let js_sha256 = require("js-sha256");
+//#region src/batteries/tools/web_retrieval/index.ts
+/**
+* RAG glue: turn web-search and web-scrape results into `Retrievable` records for a turn.
+*
+* @module @nhtio/adk/batteries/tools/web_retrieval
+*
+* @remarks
+* The seam from "I searched / scraped something" to "it is in the turn as a `Retrievable`",
+* shared by the SearXNG and Scrapper batteries. It is deliberately **decoupled** from the ADK
+* core at runtime:
+*
+* - The converters are **pure** `(payload) => RawRetrievable[]` — they build plain data objects and
+*   never instantiate a core class, so the module's only core coupling is erased `import type`.
+* - The recommended spool-artifact type travels as an **open resolver**
+*   ({@link @nhtio/adk/forge!ArtifactConstructorResolver}), never a closed string enum — a consumer's
+*   future YAML/HTML `SpooledArtifact` subclass works with no change here. The converter hands the
+*   recommendation to the caller's `spool` hook; the caller owns the actual class import.
+* - The one helper that must construct a `Retrievable` ({@link storeRetrievables}) takes the
+*   constructor via a **resolver** (constructor / sync / async / dynamic-import), exactly like the
+*   vector battery's `createVectorStore` `client`.
+*
+* Web content is `'third-party-public'` by default — a definitional constant for open-web data
+* (NOT inferred from the URL, which CONTRIBUTING Design Decision #12 forbids); override via
+* `trustTier` when you know better.
+*/
+var nowIso = () => (/* @__PURE__ */ new Date()).toISOString();
+/** A stable, unguessable id derived from a source string (URL) plus an optional prefix. */
+var stableId = (prefix, source) => {
+	const h = (0, js_sha256.sha256)(source);
+	return prefix ? `${prefix}:${h}` : h;
+};
+/** Clamp a possibly-unbounded score into `[0, 1]`; drop non-finite. */
+var clampScore = (score) => {
+	if (typeof score !== "number" || !Number.isFinite(score)) return void 0;
+	if (score < 0) return 0;
+	if (score > 1) return 1;
+	return score;
+};
+/**
+* Resolve content to either an inline string or a caller-provided {@link SpooledArtifact}. When a
+* `spool` hook is supplied it is offered the recommended resolver; whatever it returns (artifact or
+* `undefined`→inline) is used.
+*/
+var resolveContent = (id, text, opts, recommended) => {
+	if (opts.spool) {
+		const artifact = opts.spool(id, text, recommended);
+		if (artifact) return artifact;
+	}
+	return text;
+};
+/**
+* Convert a SearXNG normalised payload into one {@link @nhtio/adk!RawRetrievable} per result.
+*
+* @remarks
+* Snippets are short, so `content` stays an inline string (the `spool` hook, if any, is still
+* offered the `text` recommendation). `source` is the result URL; `score` is clamped to `[0,1]`.
+*
+* @param payload - The SearXNG normalised payload (`{ results: [{ url, title, content, score }] }`).
+* @param opts - Trust tier, kind, id prefix, optional spool hook.
+* @param recommend - Optional artifact-resolver recommendations (the glue names no class itself).
+* @returns One `RawRetrievable` per result.
+*/
+var searxngResultsToRetrievables = (payload, opts = {}, recommend = {}) => {
+	const trustTier = opts.trustTier ?? "third-party-public";
+	const kind = opts.kind ?? "web-search-result";
+	const created = nowIso();
+	return (payload.results ?? []).map((r, i) => {
+		const source = r.url ?? "";
+		const id = stableId(opts.idPrefix, source || `${kind}:${i}`);
+		const text = [r.title, r.content].filter((s) => typeof s === "string").join("\n");
+		const recommended = recommend.text ?? recommend.markdown ?? recommend.json;
+		const raw = {
+			id,
+			content: recommended ? resolveContent(id, text, opts, recommended) : text,
+			trustTier,
+			kind,
+			createdAt: created,
+			updatedAt: created
+		};
+		if (source) raw.source = source;
+		const score = clampScore(r.score);
+		if (score !== void 0) raw.score = score;
+		return raw;
+	});
+};
+/**
+* Convert a Scrapper normalised article into a single {@link @nhtio/adk!RawRetrievable}.
+*
+* @remarks
+* Long article text is exactly what a reader-backed {@link @nhtio/adk!SpooledArtifact} is for: pass a
+* `spool` hook and the converter offers it the recommended artifact resolver (markdown when
+* `asMarkdown`, else text/HTML) so the model gets the right forged query tools. Without a hook,
+* content stays inline.
+*
+* @param article - The Scrapper normalised article.
+* @param opts - Trust tier, kind, id prefix, content source, markdown flag, optional spool hook.
+* @param recommend - Optional artifact-resolver recommendations.
+* @returns A single `RawRetrievable`.
+*/
+var scrapperArticleToRetrievable = (article, opts = {}, recommend = {}) => {
+	const trustTier = opts.trustTier ?? "third-party-public";
+	const kind = opts.kind ?? "web-article";
+	const created = nowIso();
+	const source = article.url ?? "";
+	const id = stableId(opts.idPrefix, source || kind);
+	const text = ((opts.contentSource ?? "textContent") === "content" ? article.content : article.textContent) ?? "";
+	const recommended = opts.asMarkdown ? recommend.markdown ?? recommend.text : recommend.text;
+	const raw = {
+		id,
+		content: recommended ? resolveContent(id, text, opts, recommended) : text,
+		trustTier,
+		kind,
+		createdAt: created,
+		updatedAt: created
+	};
+	if (source) raw.source = source;
+	return raw;
+};
+/**
+* Convert a Scrapper normalised links payload into one {@link @nhtio/adk!RawRetrievable} per link.
+*
+* @remarks
+* Each link's `text` becomes the (inline) content and its `url` the `source`. Link text is short,
+* so no spooling is applied.
+*
+* @param payload - The Scrapper normalised links payload (`{ links: [{ url, text }] }`).
+* @param opts - Trust tier, kind, id prefix.
+* @returns One `RawRetrievable` per link.
+*/
+var scrapperLinksToRetrievables = (payload, opts = {}) => {
+	const trustTier = opts.trustTier ?? "third-party-public";
+	const kind = opts.kind ?? "web-link";
+	const created = nowIso();
+	return (payload.links ?? []).map((l, i) => {
+		const source = l.url ?? "";
+		const raw = {
+			id: stableId(opts.idPrefix, source || `${kind}:${i}`),
+			content: l.text ?? source,
+			trustTier,
+			kind,
+			createdAt: created,
+			updatedAt: created
+		};
+		if (source) raw.source = source;
+		return raw;
+	});
+};
+/**
+* Resolve a {@link Resolver} of the `Retrievable` constructor (sync / async / `{ default }`).
+*
+* @remarks
+* Both a bare class and a resolver are `typeof 'function'`, and we hold `Retrievable` only as an
+* `import type` (no runtime value to duck-type against). We disambiguate by behaviour: invoking a
+* real ES class without `new` throws, so a bare constructor is caught and returned as-is; a resolver
+* invokes cleanly and yields the constructor (possibly via a Promise and/or a `{ default }`).
+*/
+var resolveRetrievableCtor = async (resolver) => {
+	if (typeof resolver !== "function") throw new TypeError("retrievable must be a constructor or a resolver returning one");
+	let resolved;
+	try {
+		resolved = resolver();
+	} catch {
+		return resolver;
+	}
+	if (require_tool_registry.isInstanceOf(resolved, "Promise", Promise)) resolved = await resolved;
+	if (resolved && typeof resolved === "object" && "default" in resolved) resolved = resolved.default;
+	if (typeof resolved === "function") return resolved;
+	return resolver;
+};
+/**
+* Construct {@link @nhtio/adk!Retrievable}s from `RawRetrievable`s and store each via `ctx`.
+*
+* @remarks
+* This is the only function here that touches a core class, and it does so through an injected
+* **resolver** (`deps.retrievable`) so the glue itself never value-imports `Retrievable`. Each
+* record's `RawRetrievable` validation (including the required `trustTier`) fires at construction.
+* For reader-backed content, the caller's `spool` hook will typically have used
+* `ctx.storeRetrievableBytes` already; this helper just persists the records into the turn.
+*
+* @param ctx - Anything with a `storeRetrievable` method (a `DispatchContext`, or a stub).
+* @param raws - The plain records from the converters.
+* @param deps - `{ retrievable }`: the `Retrievable` constructor or a resolver of it.
+* @returns The constructed `Retrievable` instances, in input order.
+*/
+var storeRetrievables = async (ctx, raws, deps) => {
+	const Ctor = await resolveRetrievableCtor(deps.retrievable);
+	const out = [];
+	for (const raw of raws) {
+		const record = new Ctor(raw);
+		await ctx.storeRetrievable(record);
+		out.push(record);
+	}
+	return out;
+};
+//#endregion
+exports.scrapperArticleToRetrievable = scrapperArticleToRetrievable;
+exports.scrapperLinksToRetrievables = scrapperLinksToRetrievables;
+exports.searxngResultsToRetrievables = searxngResultsToRetrievables;
+exports.storeRetrievables = storeRetrievables;
+
+//# sourceMappingURL=web_retrieval.cjs.map

package/batteries/tools/web_retrieval.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"web_retrieval.cjs","names":[],"sources":["../../../src/batteries/tools/web_retrieval/index.ts"],"sourcesContent":["/**\n * RAG glue: turn web-search and web-scrape results into `Retrievable` records for a turn.\n *\n * @module @nhtio/adk/batteries/tools/web_retrieval\n *\n * @remarks\n * The seam from \"I searched / scraped something\" to \"it is in the turn as a `Retrievable`\",\n * shared by the SearXNG and Scrapper batteries. It is deliberately **decoupled** from the ADK\n * core at runtime:\n *\n * - The converters are **pure** `(payload) => RawRetrievable[]` — they build plain data objects and\n *   never instantiate a core class, so the module's only core coupling is erased `import type`.\n * - The recommended spool-artifact type travels as an **open resolver**\n *   ({@link @nhtio/adk/forge!ArtifactConstructorResolver}), never a closed string enum — a consumer's\n *   future YAML/HTML `SpooledArtifact` subclass works with no change here. The converter hands the\n *   recommendation to the caller's `spool` hook; the caller owns the actual class import.\n * - The one helper that must construct a `Retrievable` ({@link storeRetrievables}) takes the\n *   constructor via a **resolver** (constructor / sync / async / dynamic-import), exactly like the\n *   vector battery's `createVectorStore` `client`.\n *\n * Web content is `'third-party-public'` by default — a definitional constant for open-web data\n * (NOT inferred from the URL, which CONTRIBUTING Design Decision #12 forbids); override via\n * `trustTier` when you know better.\n */\n\nimport { sha256 } from 'js-sha256'\nimport { isInstanceOf } from '@nhtio/adk/guards'\nimport type { SpooledArtifact } from '@nhtio/adk/spooled_artifact'\nimport type { ArtifactConstructorResolver } from '@nhtio/adk/forge'\nimport type { RawRetrievable, Retrievable, RetrievableTrustTier } from '@nhtio/adk/common'\n\n/** A constructor that builds a {@link @nhtio/adk!Retrievable} from a {@link @nhtio/adk!RawRetrievable}. */\nexport type RetrievableCtor = new (raw: RawRetrievable) => Retrievable\n\n/** A resolver of `T`: the value itself, or a (sync/async) thunk, optionally a module `{ default }`. */\nexport type Resolver<T> = T | (() => T | { default: T }) | (() => Promise<T | { default: T }>)\n\n/**\n * A reader-backed-artifact hook. Called by a converter for content that may be large; the\n * converter passes the artifact constructor it **recommends** for this content (an open\n * {@link @nhtio/adk/forge!ArtifactConstructorResolver}) so the caller can wrap with the right\n * subclass — preserving its forged query tools — using the caller's own core import. Return a\n * {@link @nhtio/adk!SpooledArtifact} to store the content reader-backed, or `undefined` to keep it\n * inline as a string.\n */\nexport type SpoolHook = (\n  id: string,\n  text: string,\n  recommended: ArtifactConstructorResolver\n) => SpooledArtifact | undefined\n\n/** Options common to every converter. */\nexport interface ToRetrievableOptions {\n  /**\n   * Trust tier for the produced records. Default `'third-party-public'` (web content is\n   * third-party by definition — this is a constant, not URL inference).\n   */\n  trustTier?: RetrievableTrustTier\n  /** Semantic `kind` label, e.g. `'web-search-result'`, `'web-article'`, `'web-links'`. */\n  kind?: string\n  /** Prefix for the stable, hashed record id (namespacing across sources). */\n  idPrefix?: string\n  /** Optional reader-backed-artifact hook for large content. See {@link SpoolHook}. */\n  spool?: SpoolHook\n}\n\n/**\n * The artifact-resolver recommendations a caller may supply so the glue names no concrete class\n * itself. Each converter asks for the relevant key; if the caller omits it, content stays inline.\n */\nexport interface ArtifactRecommendations {\n  /** Recommended for plain-text / HTML content (base `SpooledArtifact`). */\n  text?: ArtifactConstructorResolver\n  /** Recommended for markdown content (`SpooledMarkdownArtifact`). */\n  markdown?: ArtifactConstructorResolver\n  /** Recommended for JSON content (`SpooledJsonArtifact`). */\n  json?: ArtifactConstructorResolver\n}\n\nconst nowIso = (): string => new Date().toISOString()\n\n/** A stable, unguessable id derived from a source string (URL) plus an optional prefix. */\nconst stableId = (prefix: string | undefined, source: string): string => {\n  const h = sha256(source)\n  return prefix ? `${prefix}:${h}` : h\n}\n\n/** Clamp a possibly-unbounded score into `[0, 1]`; drop non-finite. */\nconst clampScore = (score: unknown): number | undefined => {\n  if (typeof score !== 'number' || !Number.isFinite(score)) return undefined\n  if (score < 0) return 0\n  if (score > 1) return 1\n  return score\n}\n\n/**\n * Resolve content to either an inline string or a caller-provided {@link SpooledArtifact}. When a\n * `spool` hook is supplied it is offered the recommended resolver; whatever it returns (artifact or\n * `undefined`→inline) is used.\n */\nconst resolveContent = (\n  id: string,\n  text: string,\n  opts: ToRetrievableOptions,\n  recommended: ArtifactConstructorResolver\n): string | SpooledArtifact => {\n  if (opts.spool) {\n    const artifact = opts.spool(id, text, recommended)\n    if (artifact) return artifact\n  }\n  return text\n}\n\n// ── SearXNG ──────────────────────────────────────────────────────────────────\n\n/** Minimal structural shape of a SearXNG normalised result the converter reads. */\nexport interface SearxngResultLike {\n  /** Result URL (becomes the record's `source`). */\n  url?: string\n  /** Result title (joined into the inline content). */\n  title?: string\n  /** Result snippet (joined into the inline content). */\n  content?: string\n  /** Relevance score (clamped to `[0,1]` on the record). */\n  score?: number\n}\n/** Minimal structural shape of a SearXNG normalised payload. */\nexport interface SearxngPayloadLike {\n  /** The result list. */\n  results?: SearxngResultLike[]\n}\n\n/**\n * Convert a SearXNG normalised payload into one {@link @nhtio/adk!RawRetrievable} per result.\n *\n * @remarks\n * Snippets are short, so `content` stays an inline string (the `spool` hook, if any, is still\n * offered the `text` recommendation). `source` is the result URL; `score` is clamped to `[0,1]`.\n *\n * @param payload - The SearXNG normalised payload (`{ results: [{ url, title, content, score }] }`).\n * @param opts - Trust tier, kind, id prefix, optional spool hook.\n * @param recommend - Optional artifact-resolver recommendations (the glue names no class itself).\n * @returns One `RawRetrievable` per result.\n */\nexport const searxngResultsToRetrievables = (\n  payload: SearxngPayloadLike,\n  opts: ToRetrievableOptions = {},\n  recommend: ArtifactRecommendations = {}\n): RawRetrievable[] => {\n  const trustTier: RetrievableTrustTier = opts.trustTier ?? 'third-party-public'\n  const kind = opts.kind ?? 'web-search-result'\n  const created = nowIso()\n  const results = payload.results ?? []\n  return results.map((r, i) => {\n    const source = r.url ?? ''\n    const id = stableId(opts.idPrefix, source || `${kind}:${i}`)\n    const text = [r.title, r.content].filter((s): s is string => typeof s === 'string').join('\\n')\n    const recommended = recommend.text ?? recommend.markdown ?? recommend.json\n    const content = recommended ? resolveContent(id, text, opts, recommended) : text\n    const raw: RawRetrievable = {\n      id,\n      content,\n      trustTier,\n      kind,\n      createdAt: created,\n      updatedAt: created,\n    }\n    if (source) raw.source = source\n    const score = clampScore(r.score)\n    if (score !== undefined) raw.score = score\n    return raw\n  })\n}\n\n// ── Scrapper: article ──────────────────────────────────────────────────────────\n\n/** Minimal structural shape of a Scrapper normalised article. */\nexport interface ScrapperArticleLike {\n  /** The page URL (becomes the record's `source`). */\n  url?: string\n  /** Article title. */\n  title?: string\n  /** Article text with HTML stripped (the default content source). */\n  textContent?: string\n  /** Processed article HTML (the `'content'` content source). */\n  content?: string\n}\n\n/** Which article text field to use as the record content. */\nexport type ArticleContentSource = 'textContent' | 'content'\n\n/** Options for {@link scrapperArticleToRetrievable}. */\nexport interface ArticleToRetrievableOptions extends ToRetrievableOptions {\n  /** Which field to use as content (default `'textContent'`). `'content'` is HTML. */\n  contentSource?: ArticleContentSource\n  /**\n   * Whether the chosen content is markdown (recommend `markdown`) rather than plain text.\n   * Default false. Use when an output pipeline rendered the article to markdown.\n   */\n  asMarkdown?: boolean\n}\n\n/**\n * Convert a Scrapper normalised article into a single {@link @nhtio/adk!RawRetrievable}.\n *\n * @remarks\n * Long article text is exactly what a reader-backed {@link @nhtio/adk!SpooledArtifact} is for: pass a\n * `spool` hook and the converter offers it the recommended artifact resolver (markdown when\n * `asMarkdown`, else text/HTML) so the model gets the right forged query tools. Without a hook,\n * content stays inline.\n *\n * @param article - The Scrapper normalised article.\n * @param opts - Trust tier, kind, id prefix, content source, markdown flag, optional spool hook.\n * @param recommend - Optional artifact-resolver recommendations.\n * @returns A single `RawRetrievable`.\n */\nexport const scrapperArticleToRetrievable = (\n  article: ScrapperArticleLike,\n  opts: ArticleToRetrievableOptions = {},\n  recommend: ArtifactRecommendations = {}\n): RawRetrievable => {\n  const trustTier: RetrievableTrustTier = opts.trustTier ?? 'third-party-public'\n  const kind = opts.kind ?? 'web-article'\n  const created = nowIso()\n  const source = article.url ?? ''\n  const id = stableId(opts.idPrefix, source || kind)\n  const field = opts.contentSource ?? 'textContent'\n  const text = (field === 'content' ? article.content : article.textContent) ?? ''\n  const recommended = opts.asMarkdown ? (recommend.markdown ?? recommend.text) : recommend.text\n  const content = recommended ? resolveContent(id, text, opts, recommended) : text\n  const raw: RawRetrievable = {\n    id,\n    content,\n    trustTier,\n    kind,\n    createdAt: created,\n    updatedAt: created,\n  }\n  if (source) raw.source = source\n  return raw\n}\n\n// ── Scrapper: links ──────────────────────────────────────────────────────────\n\n/** Minimal structural shape of a Scrapper normalised link. */\nexport interface ScrapperLinkLike {\n  /** The link's target URL (becomes the record's `source`). */\n  url?: string\n  /** The link's anchor text (becomes the record's content). */\n  text?: string\n}\n/** Minimal structural shape of a Scrapper normalised links payload. */\nexport interface ScrapperLinksLike {\n  /** The page URL the links were collected from. */\n  url?: string\n  /** The collected links. */\n  links?: ScrapperLinkLike[]\n}\n\n/**\n * Convert a Scrapper normalised links payload into one {@link @nhtio/adk!RawRetrievable} per link.\n *\n * @remarks\n * Each link's `text` becomes the (inline) content and its `url` the `source`. Link text is short,\n * so no spooling is applied.\n *\n * @param payload - The Scrapper normalised links payload (`{ links: [{ url, text }] }`).\n * @param opts - Trust tier, kind, id prefix.\n * @returns One `RawRetrievable` per link.\n */\nexport const scrapperLinksToRetrievables = (\n  payload: ScrapperLinksLike,\n  opts: ToRetrievableOptions = {}\n): RawRetrievable[] => {\n  const trustTier: RetrievableTrustTier = opts.trustTier ?? 'third-party-public'\n  const kind = opts.kind ?? 'web-link'\n  const created = nowIso()\n  const links = payload.links ?? []\n  return links.map((l, i) => {\n    const source = l.url ?? ''\n    const id = stableId(opts.idPrefix, source || `${kind}:${i}`)\n    const raw: RawRetrievable = {\n      id,\n      content: l.text ?? source,\n      trustTier,\n      kind,\n      createdAt: created,\n      updatedAt: created,\n    }\n    if (source) raw.source = source\n    return raw\n  })\n}\n\n// ── Store helper (the single core-touching function) ─────────────────────────\n\n/** The minimal context surface {@link storeRetrievables} needs. */\nexport interface RetrievableStoreCtx {\n  /** Persist a single `Retrievable` into the turn (a `DispatchContext` method, or a stub). */\n  storeRetrievable: (v: Retrievable) => unknown | Promise<unknown>\n}\n\n/**\n * Resolve a {@link Resolver} of the `Retrievable` constructor (sync / async / `{ default }`).\n *\n * @remarks\n * Both a bare class and a resolver are `typeof 'function'`, and we hold `Retrievable` only as an\n * `import type` (no runtime value to duck-type against). We disambiguate by behaviour: invoking a\n * real ES class without `new` throws, so a bare constructor is caught and returned as-is; a resolver\n * invokes cleanly and yields the constructor (possibly via a Promise and/or a `{ default }`).\n */\nconst resolveRetrievableCtor = async (\n  resolver: Resolver<RetrievableCtor>\n): Promise<RetrievableCtor> => {\n  if (typeof resolver !== 'function') {\n    throw new TypeError('retrievable must be a constructor or a resolver returning one')\n  }\n  let resolved: unknown\n  try {\n    resolved = (resolver as () => unknown)()\n  } catch {\n    return resolver as RetrievableCtor // bare class: threw on no-`new` invocation\n  }\n  if (isInstanceOf(resolved, 'Promise', Promise)) resolved = await resolved\n  if (resolved && typeof resolved === 'object' && 'default' in resolved) {\n    resolved = (resolved as { default?: unknown }).default\n  }\n  if (typeof resolved === 'function') return resolved as RetrievableCtor\n  return resolver as RetrievableCtor // resolver returned a non-function: it was itself the ctor\n}\n\n/**\n * Construct {@link @nhtio/adk!Retrievable}s from `RawRetrievable`s and store each via `ctx`.\n *\n * @remarks\n * This is the only function here that touches a core class, and it does so through an injected\n * **resolver** (`deps.retrievable`) so the glue itself never value-imports `Retrievable`. Each\n * record's `RawRetrievable` validation (including the required `trustTier`) fires at construction.\n * For reader-backed content, the caller's `spool` hook will typically have used\n * `ctx.storeRetrievableBytes` already; this helper just persists the records into the turn.\n *\n * @param ctx - Anything with a `storeRetrievable` method (a `DispatchContext`, or a stub).\n * @param raws - The plain records from the converters.\n * @param deps - `{ retrievable }`: the `Retrievable` constructor or a resolver of it.\n * @returns The constructed `Retrievable` instances, in input order.\n */\nexport const storeRetrievables = async (\n  ctx: RetrievableStoreCtx,\n  raws: RawRetrievable[],\n  deps: { retrievable: Resolver<RetrievableCtor> }\n): Promise<Retrievable[]> => {\n  const Ctor = await resolveRetrievableCtor(deps.retrievable)\n  const out: Retrievable[] = []\n  for (const raw of raws) {\n    const record = new Ctor(raw)\n    await ctx.storeRetrievable(record)\n    out.push(record)\n  }\n  return out\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+EA,IAAM,gCAAuB,IAAI,KAAK,GAAE,YAAY;;AAGpD,IAAM,YAAY,QAA4B,WAA2B;CACvE,MAAM,KAAA,GAAA,UAAA,QAAW,MAAM;CACvB,OAAO,SAAS,GAAG,OAAO,GAAG,MAAM;AACrC;;AAGA,IAAM,cAAc,UAAuC;CACzD,IAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,GAAG,OAAO,KAAA;CACjE,IAAI,QAAQ,GAAG,OAAO;CACtB,IAAI,QAAQ,GAAG,OAAO;CACtB,OAAO;AACT;;;;;;AAOA,IAAM,kBACJ,IACA,MACA,MACA,gBAC6B;CAC7B,IAAI,KAAK,OAAO;EACd,MAAM,WAAW,KAAK,MAAM,IAAI,MAAM,WAAW;EACjD,IAAI,UAAU,OAAO;CACvB;CACA,OAAO;AACT;;;;;;;;;;;;;AAiCA,IAAa,gCACX,SACA,OAA6B,CAAC,GAC9B,YAAqC,CAAC,MACjB;CACrB,MAAM,YAAkC,KAAK,aAAa;CAC1D,MAAM,OAAO,KAAK,QAAQ;CAC1B,MAAM,UAAU,OAAO;CAEvB,QADgB,QAAQ,WAAW,CAAC,GACrB,KAAK,GAAG,MAAM;EAC3B,MAAM,SAAS,EAAE,OAAO;EACxB,MAAM,KAAK,SAAS,KAAK,UAAU,UAAU,GAAG,KAAK,GAAG,GAAG;EAC3D,MAAM,OAAO,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,MAAmB,OAAO,MAAM,QAAQ,EAAE,KAAK,IAAI;EAC7F,MAAM,cAAc,UAAU,QAAQ,UAAU,YAAY,UAAU;EAEtE,MAAM,MAAsB;GAC1B;GACA,SAHc,cAAc,eAAe,IAAI,MAAM,MAAM,WAAW,IAAI;GAI1E;GACA;GACA,WAAW;GACX,WAAW;EACb;EACA,IAAI,QAAQ,IAAI,SAAS;EACzB,MAAM,QAAQ,WAAW,EAAE,KAAK;EAChC,IAAI,UAAU,KAAA,GAAW,IAAI,QAAQ;EACrC,OAAO;CACT,CAAC;AACH;;;;;;;;;;;;;;;AA4CA,IAAa,gCACX,SACA,OAAoC,CAAC,GACrC,YAAqC,CAAC,MACnB;CACnB,MAAM,YAAkC,KAAK,aAAa;CAC1D,MAAM,OAAO,KAAK,QAAQ;CAC1B,MAAM,UAAU,OAAO;CACvB,MAAM,SAAS,QAAQ,OAAO;CAC9B,MAAM,KAAK,SAAS,KAAK,UAAU,UAAU,IAAI;CAEjD,MAAM,SADQ,KAAK,iBAAiB,mBACZ,YAAY,QAAQ,UAAU,QAAQ,gBAAgB;CAC9E,MAAM,cAAc,KAAK,aAAc,UAAU,YAAY,UAAU,OAAQ,UAAU;CAEzF,MAAM,MAAsB;EAC1B;EACA,SAHc,cAAc,eAAe,IAAI,MAAM,MAAM,WAAW,IAAI;EAI1E;EACA;EACA,WAAW;EACX,WAAW;CACb;CACA,IAAI,QAAQ,IAAI,SAAS;CACzB,OAAO;AACT;;;;;;;;;;;;AA8BA,IAAa,+BACX,SACA,OAA6B,CAAC,MACT;CACrB,MAAM,YAAkC,KAAK,aAAa;CAC1D,MAAM,OAAO,KAAK,QAAQ;CAC1B,MAAM,UAAU,OAAO;CAEvB,QADc,QAAQ,SAAS,CAAC,GACnB,KAAK,GAAG,MAAM;EACzB,MAAM,SAAS,EAAE,OAAO;EAExB,MAAM,MAAsB;GAC1B,IAFS,SAAS,KAAK,UAAU,UAAU,GAAG,KAAK,GAAG,GAEtD;GACA,SAAS,EAAE,QAAQ;GACnB;GACA;GACA,WAAW;GACX,WAAW;EACb;EACA,IAAI,QAAQ,IAAI,SAAS;EACzB,OAAO;CACT,CAAC;AACH;;;;;;;;;;AAmBA,IAAM,yBAAyB,OAC7B,aAC6B;CAC7B,IAAI,OAAO,aAAa,YACtB,MAAM,IAAI,UAAU,+DAA+D;CAErF,IAAI;CACJ,IAAI;EACF,WAAY,SAA2B;CACzC,QAAQ;EACN,OAAO;CACT;CACA,IAAI,sBAAA,aAAa,UAAU,WAAW,OAAO,GAAG,WAAW,MAAM;CACjE,IAAI,YAAY,OAAO,aAAa,YAAY,aAAa,UAC3D,WAAY,SAAmC;CAEjD,IAAI,OAAO,aAAa,YAAY,OAAO;CAC3C,OAAO;AACT;;;;;;;;;;;;;;;;AAiBA,IAAa,oBAAoB,OAC/B,KACA,MACA,SAC2B;CAC3B,MAAM,OAAO,MAAM,uBAAuB,KAAK,WAAW;CAC1D,MAAM,MAAqB,CAAC;CAC5B,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,SAAS,IAAI,KAAK,GAAG;EAC3B,MAAM,IAAI,iBAAiB,MAAM;EACjC,IAAI,KAAK,MAAM;CACjB;CACA,OAAO;AACT"}

package/batteries/tools/web_retrieval.mjs

@@ -0,0 +1,201 @@
+import { s as isInstanceOf } from "../../tool_registry-791Vrjtf.mjs";
+import "../../guards.mjs";
+import { sha256 } from "js-sha256";
+//#region src/batteries/tools/web_retrieval/index.ts
+/**
+* RAG glue: turn web-search and web-scrape results into `Retrievable` records for a turn.
+*
+* @module @nhtio/adk/batteries/tools/web_retrieval
+*
+* @remarks
+* The seam from "I searched / scraped something" to "it is in the turn as a `Retrievable`",
+* shared by the SearXNG and Scrapper batteries. It is deliberately **decoupled** from the ADK
+* core at runtime:
+*
+* - The converters are **pure** `(payload) => RawRetrievable[]` — they build plain data objects and
+*   never instantiate a core class, so the module's only core coupling is erased `import type`.
+* - The recommended spool-artifact type travels as an **open resolver**
+*   ({@link @nhtio/adk/forge!ArtifactConstructorResolver}), never a closed string enum — a consumer's
+*   future YAML/HTML `SpooledArtifact` subclass works with no change here. The converter hands the
+*   recommendation to the caller's `spool` hook; the caller owns the actual class import.
+* - The one helper that must construct a `Retrievable` ({@link storeRetrievables}) takes the
+*   constructor via a **resolver** (constructor / sync / async / dynamic-import), exactly like the
+*   vector battery's `createVectorStore` `client`.
+*
+* Web content is `'third-party-public'` by default — a definitional constant for open-web data
+* (NOT inferred from the URL, which CONTRIBUTING Design Decision #12 forbids); override via
+* `trustTier` when you know better.
+*/
+var nowIso = () => (/* @__PURE__ */ new Date()).toISOString();
+/** A stable, unguessable id derived from a source string (URL) plus an optional prefix. */
+var stableId = (prefix, source) => {
+	const h = sha256(source);
+	return prefix ? `${prefix}:${h}` : h;
+};
+/** Clamp a possibly-unbounded score into `[0, 1]`; drop non-finite. */
+var clampScore = (score) => {
+	if (typeof score !== "number" || !Number.isFinite(score)) return void 0;
+	if (score < 0) return 0;
+	if (score > 1) return 1;
+	return score;
+};
+/**
+* Resolve content to either an inline string or a caller-provided {@link SpooledArtifact}. When a
+* `spool` hook is supplied it is offered the recommended resolver; whatever it returns (artifact or
+* `undefined`→inline) is used.
+*/
+var resolveContent = (id, text, opts, recommended) => {
+	if (opts.spool) {
+		const artifact = opts.spool(id, text, recommended);
+		if (artifact) return artifact;
+	}
+	return text;
+};
+/**
+* Convert a SearXNG normalised payload into one {@link @nhtio/adk!RawRetrievable} per result.
+*
+* @remarks
+* Snippets are short, so `content` stays an inline string (the `spool` hook, if any, is still
+* offered the `text` recommendation). `source` is the result URL; `score` is clamped to `[0,1]`.
+*
+* @param payload - The SearXNG normalised payload (`{ results: [{ url, title, content, score }] }`).
+* @param opts - Trust tier, kind, id prefix, optional spool hook.
+* @param recommend - Optional artifact-resolver recommendations (the glue names no class itself).
+* @returns One `RawRetrievable` per result.
+*/
+var searxngResultsToRetrievables = (payload, opts = {}, recommend = {}) => {
+	const trustTier = opts.trustTier ?? "third-party-public";
+	const kind = opts.kind ?? "web-search-result";
+	const created = nowIso();
+	return (payload.results ?? []).map((r, i) => {
+		const source = r.url ?? "";
+		const id = stableId(opts.idPrefix, source || `${kind}:${i}`);
+		const text = [r.title, r.content].filter((s) => typeof s === "string").join("\n");
+		const recommended = recommend.text ?? recommend.markdown ?? recommend.json;
+		const raw = {
+			id,
+			content: recommended ? resolveContent(id, text, opts, recommended) : text,
+			trustTier,
+			kind,
+			createdAt: created,
+			updatedAt: created
+		};
+		if (source) raw.source = source;
+		const score = clampScore(r.score);
+		if (score !== void 0) raw.score = score;
+		return raw;
+	});
+};
+/**
+* Convert a Scrapper normalised article into a single {@link @nhtio/adk!RawRetrievable}.
+*
+* @remarks
+* Long article text is exactly what a reader-backed {@link @nhtio/adk!SpooledArtifact} is for: pass a
+* `spool` hook and the converter offers it the recommended artifact resolver (markdown when
+* `asMarkdown`, else text/HTML) so the model gets the right forged query tools. Without a hook,
+* content stays inline.
+*
+* @param article - The Scrapper normalised article.
+* @param opts - Trust tier, kind, id prefix, content source, markdown flag, optional spool hook.
+* @param recommend - Optional artifact-resolver recommendations.
+* @returns A single `RawRetrievable`.
+*/
+var scrapperArticleToRetrievable = (article, opts = {}, recommend = {}) => {
+	const trustTier = opts.trustTier ?? "third-party-public";
+	const kind = opts.kind ?? "web-article";
+	const created = nowIso();
+	const source = article.url ?? "";
+	const id = stableId(opts.idPrefix, source || kind);
+	const text = ((opts.contentSource ?? "textContent") === "content" ? article.content : article.textContent) ?? "";
+	const recommended = opts.asMarkdown ? recommend.markdown ?? recommend.text : recommend.text;
+	const raw = {
+		id,
+		content: recommended ? resolveContent(id, text, opts, recommended) : text,
+		trustTier,
+		kind,
+		createdAt: created,
+		updatedAt: created
+	};
+	if (source) raw.source = source;
+	return raw;
+};
+/**
+* Convert a Scrapper normalised links payload into one {@link @nhtio/adk!RawRetrievable} per link.
+*
+* @remarks
+* Each link's `text` becomes the (inline) content and its `url` the `source`. Link text is short,
+* so no spooling is applied.
+*
+* @param payload - The Scrapper normalised links payload (`{ links: [{ url, text }] }`).
+* @param opts - Trust tier, kind, id prefix.
+* @returns One `RawRetrievable` per link.
+*/
+var scrapperLinksToRetrievables = (payload, opts = {}) => {
+	const trustTier = opts.trustTier ?? "third-party-public";
+	const kind = opts.kind ?? "web-link";
+	const created = nowIso();
+	return (payload.links ?? []).map((l, i) => {
+		const source = l.url ?? "";
+		const raw = {
+			id: stableId(opts.idPrefix, source || `${kind}:${i}`),
+			content: l.text ?? source,
+			trustTier,
+			kind,
+			createdAt: created,
+			updatedAt: created
+		};
+		if (source) raw.source = source;
+		return raw;
+	});
+};
+/**
+* Resolve a {@link Resolver} of the `Retrievable` constructor (sync / async / `{ default }`).
+*
+* @remarks
+* Both a bare class and a resolver are `typeof 'function'`, and we hold `Retrievable` only as an
+* `import type` (no runtime value to duck-type against). We disambiguate by behaviour: invoking a
+* real ES class without `new` throws, so a bare constructor is caught and returned as-is; a resolver
+* invokes cleanly and yields the constructor (possibly via a Promise and/or a `{ default }`).
+*/
+var resolveRetrievableCtor = async (resolver) => {
+	if (typeof resolver !== "function") throw new TypeError("retrievable must be a constructor or a resolver returning one");
+	let resolved;
+	try {
+		resolved = resolver();
+	} catch {
+		return resolver;
+	}
+	if (isInstanceOf(resolved, "Promise", Promise)) resolved = await resolved;
+	if (resolved && typeof resolved === "object" && "default" in resolved) resolved = resolved.default;
+	if (typeof resolved === "function") return resolved;
+	return resolver;
+};
+/**
+* Construct {@link @nhtio/adk!Retrievable}s from `RawRetrievable`s and store each via `ctx`.
+*
+* @remarks
+* This is the only function here that touches a core class, and it does so through an injected
+* **resolver** (`deps.retrievable`) so the glue itself never value-imports `Retrievable`. Each
+* record's `RawRetrievable` validation (including the required `trustTier`) fires at construction.
+* For reader-backed content, the caller's `spool` hook will typically have used
+* `ctx.storeRetrievableBytes` already; this helper just persists the records into the turn.
+*
+* @param ctx - Anything with a `storeRetrievable` method (a `DispatchContext`, or a stub).
+* @param raws - The plain records from the converters.
+* @param deps - `{ retrievable }`: the `Retrievable` constructor or a resolver of it.
+* @returns The constructed `Retrievable` instances, in input order.
+*/
+var storeRetrievables = async (ctx, raws, deps) => {
+	const Ctor = await resolveRetrievableCtor(deps.retrievable);
+	const out = [];
+	for (const raw of raws) {
+		const record = new Ctor(raw);
+		await ctx.storeRetrievable(record);
+		out.push(record);
+	}
+	return out;
+};
+//#endregion
+export { scrapperArticleToRetrievable, scrapperLinksToRetrievables, searxngResultsToRetrievables, storeRetrievables };
+
+//# sourceMappingURL=web_retrieval.mjs.map

package/batteries/tools/web_retrieval.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"web_retrieval.mjs","names":[],"sources":["../../../src/batteries/tools/web_retrieval/index.ts"],"sourcesContent":["/**\n * RAG glue: turn web-search and web-scrape results into `Retrievable` records for a turn.\n *\n * @module @nhtio/adk/batteries/tools/web_retrieval\n *\n * @remarks\n * The seam from \"I searched / scraped something\" to \"it is in the turn as a `Retrievable`\",\n * shared by the SearXNG and Scrapper batteries. It is deliberately **decoupled** from the ADK\n * core at runtime:\n *\n * - The converters are **pure** `(payload) => RawRetrievable[]` — they build plain data objects and\n *   never instantiate a core class, so the module's only core coupling is erased `import type`.\n * - The recommended spool-artifact type travels as an **open resolver**\n *   ({@link @nhtio/adk/forge!ArtifactConstructorResolver}), never a closed string enum — a consumer's\n *   future YAML/HTML `SpooledArtifact` subclass works with no change here. The converter hands the\n *   recommendation to the caller's `spool` hook; the caller owns the actual class import.\n * - The one helper that must construct a `Retrievable` ({@link storeRetrievables}) takes the\n *   constructor via a **resolver** (constructor / sync / async / dynamic-import), exactly like the\n *   vector battery's `createVectorStore` `client`.\n *\n * Web content is `'third-party-public'` by default — a definitional constant for open-web data\n * (NOT inferred from the URL, which CONTRIBUTING Design Decision #12 forbids); override via\n * `trustTier` when you know better.\n */\n\nimport { sha256 } from 'js-sha256'\nimport { isInstanceOf } from '@nhtio/adk/guards'\nimport type { SpooledArtifact } from '@nhtio/adk/spooled_artifact'\nimport type { ArtifactConstructorResolver } from '@nhtio/adk/forge'\nimport type { RawRetrievable, Retrievable, RetrievableTrustTier } from '@nhtio/adk/common'\n\n/** A constructor that builds a {@link @nhtio/adk!Retrievable} from a {@link @nhtio/adk!RawRetrievable}. */\nexport type RetrievableCtor = new (raw: RawRetrievable) => Retrievable\n\n/** A resolver of `T`: the value itself, or a (sync/async) thunk, optionally a module `{ default }`. */\nexport type Resolver<T> = T | (() => T | { default: T }) | (() => Promise<T | { default: T }>)\n\n/**\n * A reader-backed-artifact hook. Called by a converter for content that may be large; the\n * converter passes the artifact constructor it **recommends** for this content (an open\n * {@link @nhtio/adk/forge!ArtifactConstructorResolver}) so the caller can wrap with the right\n * subclass — preserving its forged query tools — using the caller's own core import. Return a\n * {@link @nhtio/adk!SpooledArtifact} to store the content reader-backed, or `undefined` to keep it\n * inline as a string.\n */\nexport type SpoolHook = (\n  id: string,\n  text: string,\n  recommended: ArtifactConstructorResolver\n) => SpooledArtifact | undefined\n\n/** Options common to every converter. */\nexport interface ToRetrievableOptions {\n  /**\n   * Trust tier for the produced records. Default `'third-party-public'` (web content is\n   * third-party by definition — this is a constant, not URL inference).\n   */\n  trustTier?: RetrievableTrustTier\n  /** Semantic `kind` label, e.g. `'web-search-result'`, `'web-article'`, `'web-links'`. */\n  kind?: string\n  /** Prefix for the stable, hashed record id (namespacing across sources). */\n  idPrefix?: string\n  /** Optional reader-backed-artifact hook for large content. See {@link SpoolHook}. */\n  spool?: SpoolHook\n}\n\n/**\n * The artifact-resolver recommendations a caller may supply so the glue names no concrete class\n * itself. Each converter asks for the relevant key; if the caller omits it, content stays inline.\n */\nexport interface ArtifactRecommendations {\n  /** Recommended for plain-text / HTML content (base `SpooledArtifact`). */\n  text?: ArtifactConstructorResolver\n  /** Recommended for markdown content (`SpooledMarkdownArtifact`). */\n  markdown?: ArtifactConstructorResolver\n  /** Recommended for JSON content (`SpooledJsonArtifact`). */\n  json?: ArtifactConstructorResolver\n}\n\nconst nowIso = (): string => new Date().toISOString()\n\n/** A stable, unguessable id derived from a source string (URL) plus an optional prefix. */\nconst stableId = (prefix: string | undefined, source: string): string => {\n  const h = sha256(source)\n  return prefix ? `${prefix}:${h}` : h\n}\n\n/** Clamp a possibly-unbounded score into `[0, 1]`; drop non-finite. */\nconst clampScore = (score: unknown): number | undefined => {\n  if (typeof score !== 'number' || !Number.isFinite(score)) return undefined\n  if (score < 0) return 0\n  if (score > 1) return 1\n  return score\n}\n\n/**\n * Resolve content to either an inline string or a caller-provided {@link SpooledArtifact}. When a\n * `spool` hook is supplied it is offered the recommended resolver; whatever it returns (artifact or\n * `undefined`→inline) is used.\n */\nconst resolveContent = (\n  id: string,\n  text: string,\n  opts: ToRetrievableOptions,\n  recommended: ArtifactConstructorResolver\n): string | SpooledArtifact => {\n  if (opts.spool) {\n    const artifact = opts.spool(id, text, recommended)\n    if (artifact) return artifact\n  }\n  return text\n}\n\n// ── SearXNG ──────────────────────────────────────────────────────────────────\n\n/** Minimal structural shape of a SearXNG normalised result the converter reads. */\nexport interface SearxngResultLike {\n  /** Result URL (becomes the record's `source`). */\n  url?: string\n  /** Result title (joined into the inline content). */\n  title?: string\n  /** Result snippet (joined into the inline content). */\n  content?: string\n  /** Relevance score (clamped to `[0,1]` on the record). */\n  score?: number\n}\n/** Minimal structural shape of a SearXNG normalised payload. */\nexport interface SearxngPayloadLike {\n  /** The result list. */\n  results?: SearxngResultLike[]\n}\n\n/**\n * Convert a SearXNG normalised payload into one {@link @nhtio/adk!RawRetrievable} per result.\n *\n * @remarks\n * Snippets are short, so `content` stays an inline string (the `spool` hook, if any, is still\n * offered the `text` recommendation). `source` is the result URL; `score` is clamped to `[0,1]`.\n *\n * @param payload - The SearXNG normalised payload (`{ results: [{ url, title, content, score }] }`).\n * @param opts - Trust tier, kind, id prefix, optional spool hook.\n * @param recommend - Optional artifact-resolver recommendations (the glue names no class itself).\n * @returns One `RawRetrievable` per result.\n */\nexport const searxngResultsToRetrievables = (\n  payload: SearxngPayloadLike,\n  opts: ToRetrievableOptions = {},\n  recommend: ArtifactRecommendations = {}\n): RawRetrievable[] => {\n  const trustTier: RetrievableTrustTier = opts.trustTier ?? 'third-party-public'\n  const kind = opts.kind ?? 'web-search-result'\n  const created = nowIso()\n  const results = payload.results ?? []\n  return results.map((r, i) => {\n    const source = r.url ?? ''\n    const id = stableId(opts.idPrefix, source || `${kind}:${i}`)\n    const text = [r.title, r.content].filter((s): s is string => typeof s === 'string').join('\\n')\n    const recommended = recommend.text ?? recommend.markdown ?? recommend.json\n    const content = recommended ? resolveContent(id, text, opts, recommended) : text\n    const raw: RawRetrievable = {\n      id,\n      content,\n      trustTier,\n      kind,\n      createdAt: created,\n      updatedAt: created,\n    }\n    if (source) raw.source = source\n    const score = clampScore(r.score)\n    if (score !== undefined) raw.score = score\n    return raw\n  })\n}\n\n// ── Scrapper: article ──────────────────────────────────────────────────────────\n\n/** Minimal structural shape of a Scrapper normalised article. */\nexport interface ScrapperArticleLike {\n  /** The page URL (becomes the record's `source`). */\n  url?: string\n  /** Article title. */\n  title?: string\n  /** Article text with HTML stripped (the default content source). */\n  textContent?: string\n  /** Processed article HTML (the `'content'` content source). */\n  content?: string\n}\n\n/** Which article text field to use as the record content. */\nexport type ArticleContentSource = 'textContent' | 'content'\n\n/** Options for {@link scrapperArticleToRetrievable}. */\nexport interface ArticleToRetrievableOptions extends ToRetrievableOptions {\n  /** Which field to use as content (default `'textContent'`). `'content'` is HTML. */\n  contentSource?: ArticleContentSource\n  /**\n   * Whether the chosen content is markdown (recommend `markdown`) rather than plain text.\n   * Default false. Use when an output pipeline rendered the article to markdown.\n   */\n  asMarkdown?: boolean\n}\n\n/**\n * Convert a Scrapper normalised article into a single {@link @nhtio/adk!RawRetrievable}.\n *\n * @remarks\n * Long article text is exactly what a reader-backed {@link @nhtio/adk!SpooledArtifact} is for: pass a\n * `spool` hook and the converter offers it the recommended artifact resolver (markdown when\n * `asMarkdown`, else text/HTML) so the model gets the right forged query tools. Without a hook,\n * content stays inline.\n *\n * @param article - The Scrapper normalised article.\n * @param opts - Trust tier, kind, id prefix, content source, markdown flag, optional spool hook.\n * @param recommend - Optional artifact-resolver recommendations.\n * @returns A single `RawRetrievable`.\n */\nexport const scrapperArticleToRetrievable = (\n  article: ScrapperArticleLike,\n  opts: ArticleToRetrievableOptions = {},\n  recommend: ArtifactRecommendations = {}\n): RawRetrievable => {\n  const trustTier: RetrievableTrustTier = opts.trustTier ?? 'third-party-public'\n  const kind = opts.kind ?? 'web-article'\n  const created = nowIso()\n  const source = article.url ?? ''\n  const id = stableId(opts.idPrefix, source || kind)\n  const field = opts.contentSource ?? 'textContent'\n  const text = (field === 'content' ? article.content : article.textContent) ?? ''\n  const recommended = opts.asMarkdown ? (recommend.markdown ?? recommend.text) : recommend.text\n  const content = recommended ? resolveContent(id, text, opts, recommended) : text\n  const raw: RawRetrievable = {\n    id,\n    content,\n    trustTier,\n    kind,\n    createdAt: created,\n    updatedAt: created,\n  }\n  if (source) raw.source = source\n  return raw\n}\n\n// ── Scrapper: links ──────────────────────────────────────────────────────────\n\n/** Minimal structural shape of a Scrapper normalised link. */\nexport interface ScrapperLinkLike {\n  /** The link's target URL (becomes the record's `source`). */\n  url?: string\n  /** The link's anchor text (becomes the record's content). */\n  text?: string\n}\n/** Minimal structural shape of a Scrapper normalised links payload. */\nexport interface ScrapperLinksLike {\n  /** The page URL the links were collected from. */\n  url?: string\n  /** The collected links. */\n  links?: ScrapperLinkLike[]\n}\n\n/**\n * Convert a Scrapper normalised links payload into one {@link @nhtio/adk!RawRetrievable} per link.\n *\n * @remarks\n * Each link's `text` becomes the (inline) content and its `url` the `source`. Link text is short,\n * so no spooling is applied.\n *\n * @param payload - The Scrapper normalised links payload (`{ links: [{ url, text }] }`).\n * @param opts - Trust tier, kind, id prefix.\n * @returns One `RawRetrievable` per link.\n */\nexport const scrapperLinksToRetrievables = (\n  payload: ScrapperLinksLike,\n  opts: ToRetrievableOptions = {}\n): RawRetrievable[] => {\n  const trustTier: RetrievableTrustTier = opts.trustTier ?? 'third-party-public'\n  const kind = opts.kind ?? 'web-link'\n  const created = nowIso()\n  const links = payload.links ?? []\n  return links.map((l, i) => {\n    const source = l.url ?? ''\n    const id = stableId(opts.idPrefix, source || `${kind}:${i}`)\n    const raw: RawRetrievable = {\n      id,\n      content: l.text ?? source,\n      trustTier,\n      kind,\n      createdAt: created,\n      updatedAt: created,\n    }\n    if (source) raw.source = source\n    return raw\n  })\n}\n\n// ── Store helper (the single core-touching function) ─────────────────────────\n\n/** The minimal context surface {@link storeRetrievables} needs. */\nexport interface RetrievableStoreCtx {\n  /** Persist a single `Retrievable` into the turn (a `DispatchContext` method, or a stub). */\n  storeRetrievable: (v: Retrievable) => unknown | Promise<unknown>\n}\n\n/**\n * Resolve a {@link Resolver} of the `Retrievable` constructor (sync / async / `{ default }`).\n *\n * @remarks\n * Both a bare class and a resolver are `typeof 'function'`, and we hold `Retrievable` only as an\n * `import type` (no runtime value to duck-type against). We disambiguate by behaviour: invoking a\n * real ES class without `new` throws, so a bare constructor is caught and returned as-is; a resolver\n * invokes cleanly and yields the constructor (possibly via a Promise and/or a `{ default }`).\n */\nconst resolveRetrievableCtor = async (\n  resolver: Resolver<RetrievableCtor>\n): Promise<RetrievableCtor> => {\n  if (typeof resolver !== 'function') {\n    throw new TypeError('retrievable must be a constructor or a resolver returning one')\n  }\n  let resolved: unknown\n  try {\n    resolved = (resolver as () => unknown)()\n  } catch {\n    return resolver as RetrievableCtor // bare class: threw on no-`new` invocation\n  }\n  if (isInstanceOf(resolved, 'Promise', Promise)) resolved = await resolved\n  if (resolved && typeof resolved === 'object' && 'default' in resolved) {\n    resolved = (resolved as { default?: unknown }).default\n  }\n  if (typeof resolved === 'function') return resolved as RetrievableCtor\n  return resolver as RetrievableCtor // resolver returned a non-function: it was itself the ctor\n}\n\n/**\n * Construct {@link @nhtio/adk!Retrievable}s from `RawRetrievable`s and store each via `ctx`.\n *\n * @remarks\n * This is the only function here that touches a core class, and it does so through an injected\n * **resolver** (`deps.retrievable`) so the glue itself never value-imports `Retrievable`. Each\n * record's `RawRetrievable` validation (including the required `trustTier`) fires at construction.\n * For reader-backed content, the caller's `spool` hook will typically have used\n * `ctx.storeRetrievableBytes` already; this helper just persists the records into the turn.\n *\n * @param ctx - Anything with a `storeRetrievable` method (a `DispatchContext`, or a stub).\n * @param raws - The plain records from the converters.\n * @param deps - `{ retrievable }`: the `Retrievable` constructor or a resolver of it.\n * @returns The constructed `Retrievable` instances, in input order.\n */\nexport const storeRetrievables = async (\n  ctx: RetrievableStoreCtx,\n  raws: RawRetrievable[],\n  deps: { retrievable: Resolver<RetrievableCtor> }\n): Promise<Retrievable[]> => {\n  const Ctor = await resolveRetrievableCtor(deps.retrievable)\n  const out: Retrievable[] = []\n  for (const raw of raws) {\n    const record = new Ctor(raw)\n    await ctx.storeRetrievable(record)\n    out.push(record)\n  }\n  return out\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+EA,IAAM,gCAAuB,IAAI,KAAK,GAAE,YAAY;;AAGpD,IAAM,YAAY,QAA4B,WAA2B;CACvE,MAAM,IAAI,OAAO,MAAM;CACvB,OAAO,SAAS,GAAG,OAAO,GAAG,MAAM;AACrC;;AAGA,IAAM,cAAc,UAAuC;CACzD,IAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,GAAG,OAAO,KAAA;CACjE,IAAI,QAAQ,GAAG,OAAO;CACtB,IAAI,QAAQ,GAAG,OAAO;CACtB,OAAO;AACT;;;;;;AAOA,IAAM,kBACJ,IACA,MACA,MACA,gBAC6B;CAC7B,IAAI,KAAK,OAAO;EACd,MAAM,WAAW,KAAK,MAAM,IAAI,MAAM,WAAW;EACjD,IAAI,UAAU,OAAO;CACvB;CACA,OAAO;AACT;;;;;;;;;;;;;AAiCA,IAAa,gCACX,SACA,OAA6B,CAAC,GAC9B,YAAqC,CAAC,MACjB;CACrB,MAAM,YAAkC,KAAK,aAAa;CAC1D,MAAM,OAAO,KAAK,QAAQ;CAC1B,MAAM,UAAU,OAAO;CAEvB,QADgB,QAAQ,WAAW,CAAC,GACrB,KAAK,GAAG,MAAM;EAC3B,MAAM,SAAS,EAAE,OAAO;EACxB,MAAM,KAAK,SAAS,KAAK,UAAU,UAAU,GAAG,KAAK,GAAG,GAAG;EAC3D,MAAM,OAAO,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,MAAmB,OAAO,MAAM,QAAQ,EAAE,KAAK,IAAI;EAC7F,MAAM,cAAc,UAAU,QAAQ,UAAU,YAAY,UAAU;EAEtE,MAAM,MAAsB;GAC1B;GACA,SAHc,cAAc,eAAe,IAAI,MAAM,MAAM,WAAW,IAAI;GAI1E;GACA;GACA,WAAW;GACX,WAAW;EACb;EACA,IAAI,QAAQ,IAAI,SAAS;EACzB,MAAM,QAAQ,WAAW,EAAE,KAAK;EAChC,IAAI,UAAU,KAAA,GAAW,IAAI,QAAQ;EACrC,OAAO;CACT,CAAC;AACH;;;;;;;;;;;;;;;AA4CA,IAAa,gCACX,SACA,OAAoC,CAAC,GACrC,YAAqC,CAAC,MACnB;CACnB,MAAM,YAAkC,KAAK,aAAa;CAC1D,MAAM,OAAO,KAAK,QAAQ;CAC1B,MAAM,UAAU,OAAO;CACvB,MAAM,SAAS,QAAQ,OAAO;CAC9B,MAAM,KAAK,SAAS,KAAK,UAAU,UAAU,IAAI;CAEjD,MAAM,SADQ,KAAK,iBAAiB,mBACZ,YAAY,QAAQ,UAAU,QAAQ,gBAAgB;CAC9E,MAAM,cAAc,KAAK,aAAc,UAAU,YAAY,UAAU,OAAQ,UAAU;CAEzF,MAAM,MAAsB;EAC1B;EACA,SAHc,cAAc,eAAe,IAAI,MAAM,MAAM,WAAW,IAAI;EAI1E;EACA;EACA,WAAW;EACX,WAAW;CACb;CACA,IAAI,QAAQ,IAAI,SAAS;CACzB,OAAO;AACT;;;;;;;;;;;;AA8BA,IAAa,+BACX,SACA,OAA6B,CAAC,MACT;CACrB,MAAM,YAAkC,KAAK,aAAa;CAC1D,MAAM,OAAO,KAAK,QAAQ;CAC1B,MAAM,UAAU,OAAO;CAEvB,QADc,QAAQ,SAAS,CAAC,GACnB,KAAK,GAAG,MAAM;EACzB,MAAM,SAAS,EAAE,OAAO;EAExB,MAAM,MAAsB;GAC1B,IAFS,SAAS,KAAK,UAAU,UAAU,GAAG,KAAK,GAAG,GAEtD;GACA,SAAS,EAAE,QAAQ;GACnB;GACA;GACA,WAAW;GACX,WAAW;EACb;EACA,IAAI,QAAQ,IAAI,SAAS;EACzB,OAAO;CACT,CAAC;AACH;;;;;;;;;;AAmBA,IAAM,yBAAyB,OAC7B,aAC6B;CAC7B,IAAI,OAAO,aAAa,YACtB,MAAM,IAAI,UAAU,+DAA+D;CAErF,IAAI;CACJ,IAAI;EACF,WAAY,SAA2B;CACzC,QAAQ;EACN,OAAO;CACT;CACA,IAAI,aAAa,UAAU,WAAW,OAAO,GAAG,WAAW,MAAM;CACjE,IAAI,YAAY,OAAO,aAAa,YAAY,aAAa,UAC3D,WAAY,SAAmC;CAEjD,IAAI,OAAO,aAAa,YAAY,OAAO;CAC3C,OAAO;AACT;;;;;;;;;;;;;;;;AAiBA,IAAa,oBAAoB,OAC/B,KACA,MACA,SAC2B;CAC3B,MAAM,OAAO,MAAM,uBAAuB,KAAK,WAAW;CAC1D,MAAM,MAAqB,CAAC;CAC5B,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,SAAS,IAAI,KAAK,GAAG;EAC3B,MAAM,IAAI,iBAAiB,MAAM;EACjC,IAAI,KAAK,MAAM;CACjB;CACA,OAAO;AACT"}

package/batteries/tools.cjs

@@ -12,15 +12,18 @@ const require_batteries_tools_math = require("./tools/math.cjs");
 const require_batteries_tools_memory = require("./tools/memory.cjs");
 const require_batteries_tools_parsing = require("./tools/parsing.cjs");
 const require_batteries_tools_retrievables = require("./tools/retrievables.cjs");
-const require_searxng = require("../searxng-Bkrwhwhw.js");
+const require_scrapper = require("../scrapper-BeweWurk.js");
+const require_searxng = require("../searxng-B_D--V5q.js");
 const require_batteries_tools_standing_instructions = require("./tools/standing_instructions.cjs");
 const require_batteries_tools_statistics = require("./tools/statistics.cjs");
 const require_batteries_tools_string_processing = require("./tools/string_processing.cjs");
 const require_batteries_tools_structured_data = require("./tools/structured_data.cjs");
 const require_batteries_tools_text_analysis = require("./tools/text_analysis.cjs");
 const require_batteries_tools_text_comparison = require("./tools/text_comparison.cjs");
+const require_batteries_tools_web_retrieval = require("./tools/web_retrieval.cjs");
 const require_batteries_tools_time = require("./tools/time.cjs");
 const require_batteries_tools_unit_conversion = require("./tools/unit_conversion.cjs");
+exports.E_INVALID_SCRAPPER_CONFIG = require_scrapper.E_INVALID_SCRAPPER_CONFIG;
 exports.E_INVALID_SEARXNG_CONFIG = require_searxng.E_INVALID_SEARXNG_CONFIG;
 exports.addStandingInstructionTool = require_batteries_tools_standing_instructions.addStandingInstructionTool;
 exports.calculateTool = require_batteries_tools_math.calculateTool;
@@ -31,7 +34,12 @@ exports.compareRecordsTool = require_batteries_tools_comparison.compareRecordsTo
 exports.compareValuesTool = require_batteries_tools_comparison.compareValuesTool;
 exports.convertTimeTool = require_batteries_tools_time.convertTimeTool;
 exports.convertUnitTool = require_batteries_tools_unit_conversion.convertUnitTool;
+exports.createScrapperArticleTool = require_scrapper.createScrapperArticleTool;
+exports.createScrapperArticleToolSync = require_scrapper.createScrapperArticleToolSync;
+exports.createScrapperLinksTool = require_scrapper.createScrapperLinksTool;
+exports.createScrapperLinksToolSync = require_scrapper.createScrapperLinksToolSync;
 exports.createSearxngSearchTool = require_searxng.createSearxngSearchTool;
+exports.createSearxngSearchToolSync = require_searxng.createSearxngSearchToolSync;
 exports.dateAddTool = require_batteries_tools_datetime_math.dateAddTool;
 exports.dateBusinessDaysTool = require_batteries_tools_datetime_extended.dateBusinessDaysTool;
 exports.dateCalendarInfoTool = require_batteries_tools_datetime_extended.dateCalendarInfoTool;
@@ -63,6 +71,9 @@ exports.parseKvTool = require_batteries_tools_parsing.parseKvTool;
 exports.parseYamlTool = require_batteries_tools_parsing.parseYamlTool;
 exports.removeStandingInstructionTool = require_batteries_tools_standing_instructions.removeStandingInstructionTool;
 exports.retrievableTools = require_batteries_tools_retrievables.retrievableTools;
+exports.scrapperArticleToRetrievable = require_batteries_tools_web_retrieval.scrapperArticleToRetrievable;
+exports.scrapperLinksToRetrievables = require_batteries_tools_web_retrieval.scrapperLinksToRetrievables;
+exports.searxngResultsToRetrievables = require_batteries_tools_web_retrieval.searxngResultsToRetrievables;
 exports.setOperationsTool = require_batteries_tools_data_structure.setOperationsTool;
 exports.standingInstructionTools = require_batteries_tools_standing_instructions.standingInstructionTools;
 exports.statsCorrelateTool = require_batteries_tools_statistics.statsCorrelateTool;
@@ -71,6 +82,7 @@ exports.statsHistogramTool = require_batteries_tools_statistics.statsHistogramTo
 exports.statsTransformTool = require_batteries_tools_statistics.statsTransformTool;
 exports.storeMemoryTool = require_batteries_tools_memory.storeMemoryTool;
 exports.storeRetrievableTool = require_batteries_tools_retrievables.storeRetrievableTool;
+exports.storeRetrievables = require_batteries_tools_web_retrieval.storeRetrievables;
 exports.stringExtractTool = require_batteries_tools_string_processing.stringExtractTool;
 exports.stringSimilarityTool = require_batteries_tools_text_comparison.stringSimilarityTool;
 exports.stringTransformTool = require_batteries_tools_string_processing.stringTransformTool;

package/batteries/tools.mjs

@@ -10,13 +10,15 @@ import { calculateTool, evaluateKatexTool } from "./tools/math.mjs";
 import { deleteMemoryTool, listMemoriesTool, memoryTools, storeMemoryTool, updateMemoryTool } from "./tools/memory.mjs";
 import { detectDelimiterTool, parseCsvTool, parseKvTool, parseYamlTool } from "./tools/parsing.mjs";
 import { deleteRetrievableTool, listRetrievablesTool, retrievableTools, storeRetrievableTool, updateRetrievableTool } from "./tools/retrievables.mjs";
-import { n as E_INVALID_SEARXNG_CONFIG, t as createSearxngSearchTool } from "../searxng-CyA-nEu5.mjs";
+import { a as E_INVALID_SCRAPPER_CONFIG, i as createScrapperLinksToolSync, n as createScrapperArticleToolSync, r as createScrapperLinksTool, t as createScrapperArticleTool } from "../scrapper-BHM1mCde.mjs";
+import { n as createSearxngSearchToolSync, r as E_INVALID_SEARXNG_CONFIG, t as createSearxngSearchTool } from "../searxng-BJFulNcK.mjs";
 import { addStandingInstructionTool, listStandingInstructionsTool, removeStandingInstructionTool, standingInstructionTools } from "./tools/standing_instructions.mjs";
 import { statsCorrelateTool, statsDescribeTool, statsHistogramTool, statsTransformTool } from "./tools/statistics.mjs";
 import { stringExtractTool, stringTransformTool } from "./tools/string_processing.mjs";
 import { formatTableTool, jsonFormatTool, validateFormatTool } from "./tools/structured_data.mjs";
 import { textAnalyzeTool, textLinesTool } from "./tools/text_analysis.mjs";
 import { stringSimilarityTool, textDiffTool } from "./tools/text_comparison.mjs";
+import { scrapperArticleToRetrievable, scrapperLinksToRetrievables, searxngResultsToRetrievables, storeRetrievables } from "./tools/web_retrieval.mjs";
 import { convertTimeTool, getCurrentTimeTool } from "./tools/time.mjs";
 import { convertUnitTool } from "./tools/unit_conversion.mjs";
-export { E_INVALID_SEARXNG_CONFIG, addStandingInstructionTool, calculateTool, colorAdjustTool, colorContrastTool, colorSchemeTool, compareRecordsTool, compareValuesTool, convertTimeTool, convertUnitTool, createSearxngSearchTool, dateAddTool, dateBusinessDaysTool, dateCalendarInfoTool, dateDiffTool, dateNthWeekdayTool, dateParseTool, datePeriodTool, deleteMemoryTool, deleteRetrievableTool, detectDelimiterTool, durationFormatTool, encodeTextTool, evaluateKatexTool, formatListTool, formatNumberTool, formatTableTool, geoBboxContainsTool, geoDistanceTool, geoWithinRadiusTool, getCurrentTimeTool, jsonFormatTool, jsonTransformTool, listMemoriesTool, listRetrievablesTool, listStandingInstructionsTool, memoryTools, parseCsvTool, parseKvTool, parseYamlTool, removeStandingInstructionTool, retrievableTools, setOperationsTool, standingInstructionTools, statsCorrelateTool, statsDescribeTool, statsHistogramTool, statsTransformTool, storeMemoryTool, storeRetrievableTool, stringExtractTool, stringSimilarityTool, stringTransformTool, textAnalyzeTool, textDiffTool, textEscapeTool, textLinesTool, unicodeNormalizeTool, updateMemoryTool, updateRetrievableTool, validateFormatTool };
+export { E_INVALID_SCRAPPER_CONFIG, E_INVALID_SEARXNG_CONFIG, addStandingInstructionTool, calculateTool, colorAdjustTool, colorContrastTool, colorSchemeTool, compareRecordsTool, compareValuesTool, convertTimeTool, convertUnitTool, createScrapperArticleTool, createScrapperArticleToolSync, createScrapperLinksTool, createScrapperLinksToolSync, createSearxngSearchTool, createSearxngSearchToolSync, dateAddTool, dateBusinessDaysTool, dateCalendarInfoTool, dateDiffTool, dateNthWeekdayTool, dateParseTool, datePeriodTool, deleteMemoryTool, deleteRetrievableTool, detectDelimiterTool, durationFormatTool, encodeTextTool, evaluateKatexTool, formatListTool, formatNumberTool, formatTableTool, geoBboxContainsTool, geoDistanceTool, geoWithinRadiusTool, getCurrentTimeTool, jsonFormatTool, jsonTransformTool, listMemoriesTool, listRetrievablesTool, listStandingInstructionsTool, memoryTools, parseCsvTool, parseKvTool, parseYamlTool, removeStandingInstructionTool, retrievableTools, scrapperArticleToRetrievable, scrapperLinksToRetrievables, searxngResultsToRetrievables, setOperationsTool, standingInstructionTools, statsCorrelateTool, statsDescribeTool, statsHistogramTool, statsTransformTool, storeMemoryTool, storeRetrievableTool, storeRetrievables, stringExtractTool, stringSimilarityTool, stringTransformTool, textAnalyzeTool, textDiffTool, textEscapeTool, textLinesTool, unicodeNormalizeTool, updateMemoryTool, updateRetrievableTool, validateFormatTool };

package/batteries.cjs

@@ -24,13 +24,15 @@ const require_batteries_tools_math = require("./batteries/tools/math.cjs");
 const require_batteries_tools_memory = require("./batteries/tools/memory.cjs");
 const require_batteries_tools_parsing = require("./batteries/tools/parsing.cjs");
 const require_batteries_tools_retrievables = require("./batteries/tools/retrievables.cjs");
-const require_searxng = require("./searxng-Bkrwhwhw.js");
+const require_scrapper = require("./scrapper-BeweWurk.js");
+const require_searxng = require("./searxng-B_D--V5q.js");
 const require_batteries_tools_standing_instructions = require("./batteries/tools/standing_instructions.cjs");
 const require_batteries_tools_statistics = require("./batteries/tools/statistics.cjs");
 const require_batteries_tools_string_processing = require("./batteries/tools/string_processing.cjs");
 const require_batteries_tools_structured_data = require("./batteries/tools/structured_data.cjs");
 const require_batteries_tools_text_analysis = require("./batteries/tools/text_analysis.cjs");
 const require_batteries_tools_text_comparison = require("./batteries/tools/text_comparison.cjs");
+const require_batteries_tools_web_retrieval = require("./batteries/tools/web_retrieval.cjs");
 const require_batteries_tools_time = require("./batteries/tools/time.cjs");
 const require_batteries_tools_unit_conversion = require("./batteries/tools/unit_conversion.cjs");
 require("./batteries/tools.cjs");
@@ -58,6 +60,7 @@ exports.CollectionBuilder = require_batteries_vector_schema.CollectionBuilder;
 exports.E_INVALID_OLLAMA_OPTIONS = require_batteries_llm_ollama_exceptions.E_INVALID_OLLAMA_OPTIONS;
 exports.E_INVALID_OPENAI_CHAT_COMPLETIONS_OPTIONS = require_batteries_llm_openai_chat_completions_exceptions.E_INVALID_OPENAI_CHAT_COMPLETIONS_OPTIONS;
 exports.E_INVALID_OPENAI_EMBEDDINGS_OPTIONS = require_batteries_embeddings_openai_exceptions.E_INVALID_OPENAI_EMBEDDINGS_OPTIONS;
+exports.E_INVALID_SCRAPPER_CONFIG = require_scrapper.E_INVALID_SCRAPPER_CONFIG;
 exports.E_INVALID_SEARXNG_CONFIG = require_searxng.E_INVALID_SEARXNG_CONFIG;
 exports.E_INVALID_VECTOR_RECORD = require_batteries_vector_exceptions.E_INVALID_VECTOR_RECORD;
 exports.E_INVALID_VECTOR_STORE_CONFIG = require_batteries_vector_exceptions.E_INVALID_VECTOR_STORE_CONFIG;
@@ -121,7 +124,12 @@ exports.compareValuesTool = require_batteries_tools_comparison.compareValuesTool
 exports.convertTimeTool = require_batteries_tools_time.convertTimeTool;
 exports.convertUnitTool = require_batteries_tools_unit_conversion.convertUnitTool;
 exports.createChatCompletionsToolCallDeltaAccumulator = require_batteries_llm_openai_chat_completions_helpers.createChatCompletionsToolCallDeltaAccumulator;
+exports.createScrapperArticleTool = require_scrapper.createScrapperArticleTool;
+exports.createScrapperArticleToolSync = require_scrapper.createScrapperArticleToolSync;
+exports.createScrapperLinksTool = require_scrapper.createScrapperLinksTool;
+exports.createScrapperLinksToolSync = require_scrapper.createScrapperLinksToolSync;
 exports.createSearxngSearchTool = require_searxng.createSearxngSearchTool;
+exports.createSearxngSearchToolSync = require_searxng.createSearxngSearchToolSync;
 exports.createVectorRetrievableCallbacks = require_batteries_vector_retrievable.createVectorRetrievableCallbacks;
 exports.createVectorStore = require_batteries_vector_factory.createVectorStore;
 exports.dateAddTool = require_batteries_tools_datetime_math.dateAddTool;
@@ -205,6 +213,9 @@ exports.renderTrustedContent = require_helpers.renderTrustedContent;
 exports.renderUntrustedContent = require_helpers.renderUntrustedContent;
 exports.resolveClientCtor = require_batteries_vector_validation.resolveClientCtor;
 exports.retrievableTools = require_batteries_tools_retrievables.retrievableTools;
+exports.scrapperArticleToRetrievable = require_batteries_tools_web_retrieval.scrapperArticleToRetrievable;
+exports.scrapperLinksToRetrievables = require_batteries_tools_web_retrieval.scrapperLinksToRetrievables;
+exports.searxngResultsToRetrievables = require_batteries_tools_web_retrieval.searxngResultsToRetrievables;
 exports.setOperationsTool = require_batteries_tools_data_structure.setOperationsTool;
 exports.standingInstructionTools = require_batteries_tools_standing_instructions.standingInstructionTools;
 exports.statsCorrelateTool = require_batteries_tools_statistics.statsCorrelateTool;
@@ -213,6 +224,7 @@ exports.statsHistogramTool = require_batteries_tools_statistics.statsHistogramTo
 exports.statsTransformTool = require_batteries_tools_statistics.statsTransformTool;
 exports.storeMemoryTool = require_batteries_tools_memory.storeMemoryTool;
 exports.storeRetrievableTool = require_batteries_tools_retrievables.storeRetrievableTool;
+exports.storeRetrievables = require_batteries_tools_web_retrieval.storeRetrievables;
 exports.stringExtractTool = require_batteries_tools_string_processing.stringExtractTool;
 exports.stringSimilarityTool = require_batteries_tools_text_comparison.stringSimilarityTool;
 exports.stringTransformTool = require_batteries_tools_string_processing.stringTransformTool;