chat-agent-toolkit 1.2.33 → 1.2.35

package/src/tools/search/doc-utils.ts

@@ -1,139 +1,139 @@
-/**
- * @module research/search/doc-utils
- * @description Document utilities: fallback docs, reranking, and formatting.
- */
-import type { Document } from "./document";
-
-export interface R2CredentialsInput {
-  accountId: string;
-  accessKeyId: string;
-  secretAccessKey: string;
-  bucket: string;
-}
-
-export function buildFallbackDocs(query: string): Document[] {
-  const trimmedQuery = (query || "").trim();
-  const searchQuery = trimmedQuery.length > 0 ? trimmedQuery : "web search";
-  const fallbackUrl = `https://www.google.com/search?q=${encodeURIComponent(searchQuery)}`;
-
-  return [
-    {
-      pageContent:
-        "No indexed sources were returned by the configured search providers for this query.",
-      metadata: {
-        title: `Search results for: ${searchQuery}`,
-        url: fallbackUrl,
-        source: "Google Search",
-      },
-    },
-  ];
-}
-
-export function normalizeSourcesOutput(output: unknown, query: string): Document[] {
-  if (Array.isArray(output) && output.length > 0) {
-    return output as Document[];
-  }
-  return buildFallbackDocs(query);
-}
-
-/**
- * Resolves an uploaded fileId to its extracted `{ title, content }` payload.
- * Hosts register a loader (e.g. reading the native R2 binding) so the search
- * pipeline does not depend on filesystem or credential-based access.
- */
-export type UploadFileLoader = (
-  fileId: string,
-) => Promise<{ title: string; content: string } | null>;
-
-let uploadFileLoader: UploadFileLoader | null = null;
-
-/**
- * Registers the loader used by {@link rerankDocs} to resolve uploaded
- * fileIds to extracted content. The registered loader takes precedence over
- * the S3-credentials fallback.
- */
-export function registerUploadFileLoader(loader: UploadFileLoader): void {
-  uploadFileLoader = loader;
-}
-
-async function downloadExtractedContent(fileId: string, r2Credentials: R2CredentialsInput): Promise<{ title: string; content: string } | null> {
-  try {
-    const { manageStorage } = await import("manage-storage");
-    const config = {
-      provider: "cloudflare" as const,
-      BUCKET_NAME: r2Credentials.bucket,
-      ACCESS_KEY_ID: r2Credentials.accessKeyId,
-      SECRET_ACCESS_KEY: r2Credentials.secretAccessKey,
-      BUCKET_URL: `https://${r2Credentials.accountId}.r2.cloudflarestorage.com`,
-    };
-    const extractedKey = `${fileId}-extracted.json`;
-    const data = await manageStorage("download", { ...config, key: extractedKey });
-    const parsed = JSON.parse(data);
-    return { title: parsed.title || "Uploaded Document", content: parsed.content || "" };
-  } catch (error) {
-    console.error(`[rerankDocs] Failed to download extracted content for fileId ${fileId}:`, error);
-    return null;
-  }
-}
-
-export async function rerankDocs(
-  query: string,
-  docs: Document[],
-  fileIds: string[],
-  optimizationMode: "speed" | "balanced" | "quality",
-  r2Credentials?: R2CredentialsInput,
-): Promise<Document[]> {
-  if (docs.length === 0 && fileIds.length === 0) {
-    return docs;
-  }
-
-  let filesData: { title: string; content: string }[] = [];
-
-  if (fileIds.length > 0) {
-    const results = await Promise.all(
-      fileIds.map(async (fileId) => {
-        if (uploadFileLoader) {
-          try {
-            const loaded = await uploadFileLoader(fileId);
-            if (loaded) return loaded;
-          } catch (error) {
-            console.error(
-              `[rerankDocs] Registered upload loader failed for fileId ${fileId}:`,
-              error,
-            );
-          }
-        }
-        if (r2Credentials) {
-          return downloadExtractedContent(fileId, r2Credentials);
-        }
-        return null;
-      }),
-    );
-    filesData = results.filter((r) => r !== null);
-  }
-
-  if (query.toLocaleLowerCase() === "summarize") {
-    return docs.slice(0, 15);
-  }
-
-  const docsWithContent = docs.filter(
-    (doc) => doc.pageContent && doc.pageContent.length > 0,
-  );
-
-  const fileDocs: Document[] = filesData.map((fileData) => ({
-    pageContent: fileData.content,
-    metadata: { title: fileData.title, url: "File" },
-  }));
-
-  // Combine file docs with web results, cap at 15
-  return [...fileDocs, ...docsWithContent].slice(0, 15);
-}
-
-export function processDocs(docs: Document[]): string {
-  return docs
-    .map(
-      (_, index) =>
-        `${index + 1}. ${docs[index].metadata.title} ${docs[index].pageContent}`,
-    )
-    .join("\n");
-}
+/**
+ * @module research/search/doc-utils
+ * @description Document utilities: fallback docs, reranking, and formatting.
+ */
+import type { Document } from "./document";
+
+export interface R2CredentialsInput {
+  accountId: string;
+  accessKeyId: string;
+  secretAccessKey: string;
+  bucket: string;
+}
+
+export function buildFallbackDocs(query: string): Document[] {
+  const trimmedQuery = (query || "").trim();
+  const searchQuery = trimmedQuery.length > 0 ? trimmedQuery : "web search";
+  const fallbackUrl = `https://www.google.com/search?q=${encodeURIComponent(searchQuery)}`;
+
+  return [
+    {
+      pageContent:
+        "No indexed sources were returned by the configured search providers for this query.",
+      metadata: {
+        title: `Search results for: ${searchQuery}`,
+        url: fallbackUrl,
+        source: "Google Search",
+      },
+    },
+  ];
+}
+
+export function normalizeSourcesOutput(output: unknown, query: string): Document[] {
+  if (Array.isArray(output) && output.length > 0) {
+    return output as Document[];
+  }
+  return buildFallbackDocs(query);
+}
+
+/**
+ * Resolves an uploaded fileId to its extracted `{ title, content }` payload.
+ * Hosts register a loader (e.g. reading the native R2 binding) so the search
+ * pipeline does not depend on filesystem or credential-based access.
+ */
+export type UploadFileLoader = (
+  fileId: string,
+) => Promise<{ title: string; content: string } | null>;
+
+let uploadFileLoader: UploadFileLoader | null = null;
+
+/**
+ * Registers the loader used by {@link rerankDocs} to resolve uploaded
+ * fileIds to extracted content. The registered loader takes precedence over
+ * the S3-credentials fallback.
+ */
+export function registerUploadFileLoader(loader: UploadFileLoader): void {
+  uploadFileLoader = loader;
+}
+
+async function downloadExtractedContent(fileId: string, r2Credentials: R2CredentialsInput): Promise<{ title: string; content: string } | null> {
+  try {
+    const { manageStorage } = await import("manage-storage");
+    const config = {
+      provider: "cloudflare" as const,
+      BUCKET_NAME: r2Credentials.bucket,
+      ACCESS_KEY_ID: r2Credentials.accessKeyId,
+      SECRET_ACCESS_KEY: r2Credentials.secretAccessKey,
+      BUCKET_URL: `https://${r2Credentials.accountId}.r2.cloudflarestorage.com`,
+    };
+    const extractedKey = `${fileId}-extracted.json`;
+    const data = await manageStorage("download", { ...config, key: extractedKey });
+    const parsed = JSON.parse(data);
+    return { title: parsed.title || "Uploaded Document", content: parsed.content || "" };
+  } catch (error) {
+    console.error(`[rerankDocs] Failed to download extracted content for fileId ${fileId}:`, error);
+    return null;
+  }
+}
+
+export async function rerankDocs(
+  query: string,
+  docs: Document[],
+  fileIds: string[],
+  optimizationMode: "speed" | "balanced" | "quality",
+  r2Credentials?: R2CredentialsInput,
+): Promise<Document[]> {
+  if (docs.length === 0 && fileIds.length === 0) {
+    return docs;
+  }
+
+  let filesData: { title: string; content: string }[] = [];
+
+  if (fileIds.length > 0) {
+    const results = await Promise.all(
+      fileIds.map(async (fileId) => {
+        if (uploadFileLoader) {
+          try {
+            const loaded = await uploadFileLoader(fileId);
+            if (loaded) return loaded;
+          } catch (error) {
+            console.error(
+              `[rerankDocs] Registered upload loader failed for fileId ${fileId}:`,
+              error,
+            );
+          }
+        }
+        if (r2Credentials) {
+          return downloadExtractedContent(fileId, r2Credentials);
+        }
+        return null;
+      }),
+    );
+    filesData = results.filter((r) => r !== null);
+  }
+
+  if (query.toLocaleLowerCase() === "summarize") {
+    return docs.slice(0, 15);
+  }
+
+  const docsWithContent = docs.filter(
+    (doc) => doc.pageContent && doc.pageContent.length > 0,
+  );
+
+  const fileDocs: Document[] = filesData.map((fileData) => ({
+    pageContent: fileData.content,
+    metadata: { title: fileData.title, url: "File" },
+  }));
+
+  // Combine file docs with web results, cap at 15
+  return [...fileDocs, ...docsWithContent].slice(0, 15);
+}
+
+export function processDocs(docs: Document[]): string {
+  return docs
+    .map(
+      (_, index) =>
+        `${index + 1}. ${docs[index].metadata.title} ${docs[index].pageContent}`,
+    )
+    .join("\n");
+}

package/src/tools/search/document.ts

@@ -1,47 +1,47 @@
-/**
- * @module research/search/document
- * @description Minimal document shape shared across the search pipeline.
- * A document is a chunk of page content plus citation metadata (title, url, source, etc.).
- */
-
-export interface Document<
-  Metadata extends Record<string, any> = Record<string, any>,
-> {
-  pageContent: string;
-  metadata: Metadata;
-}
-
-/**
- * Splits text into overlapping chunks on whitespace boundaries.
- * Default chunkSize 1000, chunkOverlap 200.
- */
-export function splitTextIntoChunks(
-  text: string,
-  chunkSize = 1000,
-  chunkOverlap = 200,
-): string[] {
-  const trimmed = (text || "").trim();
-  if (trimmed.length <= chunkSize) {
-    return trimmed.length > 0 ? [trimmed] : [];
-  }
-
-  const chunks: string[] = [];
-  let start = 0;
-
-  while (start < trimmed.length) {
-    let end = Math.min(start + chunkSize, trimmed.length);
-
-    // Break on the last whitespace inside the window so words stay intact
-    if (end < trimmed.length) {
-      const lastSpace = trimmed.lastIndexOf(" ", end);
-      if (lastSpace > start) end = lastSpace;
-    }
-
-    chunks.push(trimmed.slice(start, end).trim());
-
-    if (end >= trimmed.length) break;
-    start = Math.max(end - chunkOverlap, start + 1);
-  }
-
-  return chunks.filter((chunk) => chunk.length > 0);
-}
+/**
+ * @module research/search/document
+ * @description Minimal document shape shared across the search pipeline.
+ * A document is a chunk of page content plus citation metadata (title, url, source, etc.).
+ */
+
+export interface Document<
+  Metadata extends Record<string, any> = Record<string, any>,
+> {
+  pageContent: string;
+  metadata: Metadata;
+}
+
+/**
+ * Splits text into overlapping chunks on whitespace boundaries.
+ * Default chunkSize 1000, chunkOverlap 200.
+ */
+export function splitTextIntoChunks(
+  text: string,
+  chunkSize = 1000,
+  chunkOverlap = 200,
+): string[] {
+  const trimmed = (text || "").trim();
+  if (trimmed.length <= chunkSize) {
+    return trimmed.length > 0 ? [trimmed] : [];
+  }
+
+  const chunks: string[] = [];
+  let start = 0;
+
+  while (start < trimmed.length) {
+    let end = Math.min(start + chunkSize, trimmed.length);
+
+    // Break on the last whitespace inside the window so words stay intact
+    if (end < trimmed.length) {
+      const lastSpace = trimmed.lastIndexOf(" ", end);
+      if (lastSpace > start) end = lastSpace;
+    }
+
+    chunks.push(trimmed.slice(start, end).trim());
+
+    if (end >= trimmed.length) break;
+    start = Math.max(end - chunkOverlap, start + 1);
+  }
+
+  return chunks.filter((chunk) => chunk.length > 0);
+}

package/src/tools/search/index.ts

@@ -1,14 +1,14 @@
-/**
- * @module agent-toolkit/tools/search
- * @description Meta search agent tools for AI-powered web search and research
- */
-
-export { default as MetaSearchAgent } from "./metaSearchAgent";
-export type { MetaSearchAgentType, Config, ChatTurnMessage, SearchingEvent, FewShotExample } from "./meta-search-types";
-export { searchHandlers, createSearchHandlers } from "./search-handlers";
-export { default as generateSuggestions } from "./suggestionGeneratorAgent";
-export { groupAndSummarizeDocs } from "./link-summarizer";
-export type { Document } from "./document";
-export { splitTextIntoChunks } from "./document";
-export { buildFallbackDocs, rerankDocs, processDocs, normalizeSourcesOutput, registerUploadFileLoader } from "./doc-utils";
-export type { UploadFileLoader } from "./doc-utils";
+/**
+ * @module agent-toolkit/tools/search
+ * @description Meta search agent tools for AI-powered web search and research
+ */
+
+export { default as MetaSearchAgent } from "./metaSearchAgent";
+export type { MetaSearchAgentType, Config, ChatTurnMessage, SearchingEvent, FewShotExample } from "./meta-search-types";
+export { searchHandlers, createSearchHandlers } from "./search-handlers";
+export { default as generateSuggestions } from "./suggestionGeneratorAgent";
+export { groupAndSummarizeDocs } from "./link-summarizer";
+export type { Document } from "./document";
+export { splitTextIntoChunks } from "./document";
+export { buildFallbackDocs, rerankDocs, processDocs, normalizeSourcesOutput, registerUploadFileLoader } from "./doc-utils";
+export type { UploadFileLoader } from "./doc-utils";

package/src/tools/search/link-summarizer.ts

@@ -1,112 +1,112 @@
-/**
- * @module research/search/link-summarizer
- * @description Groups link documents by URL then summarizes each group with an LLM.
- */
-import { generateText, type LanguageModel } from "ai";
-import type { Document } from "./document";
-import { buildFallbackDocs } from "./doc-utils";
-
-const SUMMARIZER_PROMPT = `You are a web search summarizer, tasked with summarizing a piece of text retrieved from a web search. Your job is to summarize the
-text into a detailed, 2-4 paragraph explanation that captures the main ideas and provides a comprehensive answer to the query.
-If the query is "summarize", you should provide a detailed summary of the text. If the query is a specific question, you should answer it in the summary.
-
-- **Journalistic tone**: The summary should sound professional and journalistic, not too casual or vague.
-- **Thorough and detailed**: Ensure that every key point from the text is captured and that the summary directly answers the query.
-- **Not too lengthy, but detailed**: The summary should be informative but not excessively long. Focus on providing detailed information in a concise format.
-
-The text will be shared inside the \`text\` XML tag, and the query inside the \`query\` XML tag.
-
-<example>
-1. \`<text>
-Docker is a set of platform-as-a-service products that use OS-level virtualization to deliver software in packages called containers.
-It was first released in 2013 and is developed by Docker, Inc. Docker is designed to make it easier to create, deploy, and run applications
-by using containers.
-</text>
-
-<query>
-What is Docker and how does it work?
-</query>
-
-Response:
-Docker is a revolutionary platform-as-a-service product developed by Docker, Inc., that uses container technology to make application
-deployment more efficient. It allows developers to package their software with all necessary dependencies, making it easier to run in
-any environment. Released in 2013, Docker has transformed the way applications are built, deployed, and managed.
-\`
-2. \`<text>
-The theory of relativity, or simply relativity, encompasses two interrelated theories of Albert Einstein: special relativity and general
-relativity. However, the word "relativity" is sometimes used in reference to Galilean invariance. The term "theory of relativity" was based
-on the expression "relative theory" used by Max Planck in 1906. The theory of relativity usually encompasses two interrelated theories by
-Albert Einstein: special relativity and general relativity. Special relativity applies to all physical phenomena in the absence of gravity.
-General relativity explains the law of gravitation and its relation to other forces of nature. It applies to the cosmological and astrophysical
-realm, including astronomy.
-</text>
-
-<query>
-summarize
-</query>
-
-Response:
-The theory of relativity, developed by Albert Einstein, encompasses two main theories: special relativity and general relativity. Special
-relativity applies to all physical phenomena in the absence of gravity, while general relativity explains the law of gravitation and its
-relation to other forces of nature. The theory of relativity is based on the concept of "relative theory," as introduced by Max Planck in
-1906. It is a fundamental theory in physics that has revolutionized our understanding of the universe.
-\`
-</example>
-
-Everything below is the actual data you will be working with. Good luck!
-
-<query>
-{question}
-</query>
-
-<text>
-{content}
-</text>
-
-Make sure to answer the query in the summary.`;
-
-/**
- * Groups fetched link documents by URL (max 10 chunks per URL), then
- * summarizes each group with the LLM in parallel.
- */
-export async function groupAndSummarizeDocs(
-  llm: LanguageModel,
-  linkDocs: Document[],
-  question: string,
-): Promise<Document[]> {
-  // Group chunks by URL, capping at 10 chunks each
-  const docGroups: Document[] = [];
-
-  for (const doc of linkDocs) {
-    const existing = docGroups.find(
-      (d) => d.metadata.url === doc.metadata.url && d.metadata.totalDocs < 10,
-    );
-
-    if (!existing) {
-      docGroups.push({ ...doc, metadata: { ...doc.metadata, totalDocs: 1 } });
-    } else {
-      existing.pageContent += `\n\n${doc.pageContent}`;
-      existing.metadata.totalDocs += 1;
-    }
-  }
-
-  // Summarize each group in parallel
-  const docs: Document[] = [];
-
-  await Promise.all(
-    docGroups.map(async (doc) => {
-      const prompt = SUMMARIZER_PROMPT.replace("{question}", question).replace(
-        "{content}",
-        doc.pageContent,
-      );
-      const { text } = await generateText({ model: llm, prompt });
-
-      docs.push({
-        pageContent: text,
-        metadata: { title: doc.metadata.title, url: doc.metadata.url },
-      });
-    }),
-  );
-
-  return docs.length > 0 ? docs : buildFallbackDocs(question);
-}
+/**
+ * @module research/search/link-summarizer
+ * @description Groups link documents by URL then summarizes each group with an LLM.
+ */
+import { generateText, type LanguageModel } from "ai";
+import type { Document } from "./document";
+import { buildFallbackDocs } from "./doc-utils";
+
+const SUMMARIZER_PROMPT = `You are a web search summarizer, tasked with summarizing a piece of text retrieved from a web search. Your job is to summarize the
+text into a detailed, 2-4 paragraph explanation that captures the main ideas and provides a comprehensive answer to the query.
+If the query is "summarize", you should provide a detailed summary of the text. If the query is a specific question, you should answer it in the summary.
+
+- **Journalistic tone**: The summary should sound professional and journalistic, not too casual or vague.
+- **Thorough and detailed**: Ensure that every key point from the text is captured and that the summary directly answers the query.
+- **Not too lengthy, but detailed**: The summary should be informative but not excessively long. Focus on providing detailed information in a concise format.
+
+The text will be shared inside the \`text\` XML tag, and the query inside the \`query\` XML tag.
+
+<example>
+1. \`<text>
+Docker is a set of platform-as-a-service products that use OS-level virtualization to deliver software in packages called containers.
+It was first released in 2013 and is developed by Docker, Inc. Docker is designed to make it easier to create, deploy, and run applications
+by using containers.
+</text>
+
+<query>
+What is Docker and how does it work?
+</query>
+
+Response:
+Docker is a revolutionary platform-as-a-service product developed by Docker, Inc., that uses container technology to make application
+deployment more efficient. It allows developers to package their software with all necessary dependencies, making it easier to run in
+any environment. Released in 2013, Docker has transformed the way applications are built, deployed, and managed.
+\`
+2. \`<text>
+The theory of relativity, or simply relativity, encompasses two interrelated theories of Albert Einstein: special relativity and general
+relativity. However, the word "relativity" is sometimes used in reference to Galilean invariance. The term "theory of relativity" was based
+on the expression "relative theory" used by Max Planck in 1906. The theory of relativity usually encompasses two interrelated theories by
+Albert Einstein: special relativity and general relativity. Special relativity applies to all physical phenomena in the absence of gravity.
+General relativity explains the law of gravitation and its relation to other forces of nature. It applies to the cosmological and astrophysical
+realm, including astronomy.
+</text>
+
+<query>
+summarize
+</query>
+
+Response:
+The theory of relativity, developed by Albert Einstein, encompasses two main theories: special relativity and general relativity. Special
+relativity applies to all physical phenomena in the absence of gravity, while general relativity explains the law of gravitation and its
+relation to other forces of nature. The theory of relativity is based on the concept of "relative theory," as introduced by Max Planck in
+1906. It is a fundamental theory in physics that has revolutionized our understanding of the universe.
+\`
+</example>
+
+Everything below is the actual data you will be working with. Good luck!
+
+<query>
+{question}
+</query>
+
+<text>
+{content}
+</text>
+
+Make sure to answer the query in the summary.`;
+
+/**
+ * Groups fetched link documents by URL (max 10 chunks per URL), then
+ * summarizes each group with the LLM in parallel.
+ */
+export async function groupAndSummarizeDocs(
+  llm: LanguageModel,
+  linkDocs: Document[],
+  question: string,
+): Promise<Document[]> {
+  // Group chunks by URL, capping at 10 chunks each
+  const docGroups: Document[] = [];
+
+  for (const doc of linkDocs) {
+    const existing = docGroups.find(
+      (d) => d.metadata.url === doc.metadata.url && d.metadata.totalDocs < 10,
+    );
+
+    if (!existing) {
+      docGroups.push({ ...doc, metadata: { ...doc.metadata, totalDocs: 1 } });
+    } else {
+      existing.pageContent += `\n\n${doc.pageContent}`;
+      existing.metadata.totalDocs += 1;
+    }
+  }
+
+  // Summarize each group in parallel
+  const docs: Document[] = [];
+
+  await Promise.all(
+    docGroups.map(async (doc) => {
+      const prompt = SUMMARIZER_PROMPT.replace("{question}", question).replace(
+        "{content}",
+        doc.pageContent,
+      );
+      const { text } = await generateText({ model: llm, prompt });
+
+      docs.push({
+        pageContent: text,
+        metadata: { title: doc.metadata.title, url: doc.metadata.url },
+      });
+    }),
+  );
+
+  return docs.length > 0 ? docs : buildFallbackDocs(question);
+}