@librechat/agents 2.4.317 → 2.4.318

package/src/tools/search/test.ts

@@ -0,0 +1,159 @@
+/* eslint-disable no-console */
+// processWikipedia.ts
+import * as fs from 'fs';
+import * as path from 'path';
+import { processContent } from './content';
+
+/**
+ * Process a Wikipedia article (HTML and Markdown) and create a referenced version
+ */
+async function processWikipediaArticle(): Promise<void> {
+  try {
+    console.log('Starting Wikipedia article processing...');
+
+    // Define file paths - adapt these to your specific file locations
+    const htmlPath = path.resolve('./test.html');
+    const markdownPath = path.resolve('./test.md');
+    const outputPath = path.resolve('./output.md');
+
+    // Check if input files exist
+    if (!fs.existsSync(htmlPath)) {
+      throw new Error(`Wikipedia HTML file not found at ${htmlPath}`);
+    }
+
+    if (!fs.existsSync(markdownPath)) {
+      throw new Error(`Wikipedia Markdown file not found at ${markdownPath}`);
+    }
+
+    console.log('Reading Wikipedia article files...');
+    const html = fs.readFileSync(htmlPath, 'utf-8');
+    const markdown = fs.readFileSync(markdownPath, 'utf-8');
+
+    // Extract article title for logging
+    const titleMatch = /<h1[^>]*>([^<]+)<\/h1>/i.exec(html);
+    const articleTitle = titleMatch
+      ? titleMatch[1].trim()
+      : 'Wikipedia article';
+
+    console.log(`Processing "${articleTitle}"...`);
+
+    // Measure processing time
+    const startTime = process.hrtime();
+
+    // Process content
+    const result = processContent(html, markdown);
+
+    // Calculate processing time
+    const elapsed = process.hrtime(startTime);
+    const timeInMs = elapsed[0] * 1000 + elapsed[1] / 1000000;
+
+    // Generate reference appendix
+    const appendix = generateReferenceAppendix(result);
+
+    // Create complete output with the processed content and appendix
+    const completeOutput = result.markdown + appendix;
+
+    // Write to output file
+    fs.writeFileSync(outputPath, completeOutput);
+
+    // Print processing statistics
+    console.log('\nWikipedia article processing complete! ✓');
+    console.log('-'.repeat(60));
+    console.log(`Article: ${articleTitle}`);
+    console.log(`Processing time: ${timeInMs.toFixed(2)}ms`);
+    console.log('Media references replaced:');
+    console.log(`  - Links: ${result.links.length}`);
+    console.log(`  - Images: ${result.images.length}`);
+    console.log(`  - Videos: ${result.videos.length}`);
+    console.log(
+      `  - Total: ${result.links.length + result.images.length + result.videos.length}`
+    );
+    console.log(`Output saved to: ${outputPath}`);
+    console.log('-'.repeat(60));
+
+    // Print sample of the transformation
+    const sampleLines = result.markdown.split('\n').slice(0, 10).join('\n');
+    console.log('\nSample of transformed content:');
+    console.log('-'.repeat(30));
+    console.log(sampleLines);
+    console.log('-'.repeat(30));
+    console.log('... (continued in output file)');
+  } catch (error) {
+    console.error('Error processing Wikipedia article:', error);
+    process.exit(1);
+  }
+}
+
+/**
+ * Generate a comprehensive reference appendix with all media links
+ */
+function generateReferenceAppendix(result: {
+  links: Array<{ originalUrl: string; title?: string; text?: string }>;
+  images: Array<{ originalUrl: string; title?: string }>;
+  videos: Array<{ originalUrl: string; title?: string }>;
+}): string {
+  let appendix = '\n\n' + '---'.repeat(10) + '\n\n';
+  appendix += '# References\n\n';
+
+  if (result.links.length > 0) {
+    appendix += '## Links\n\n';
+    result.links.forEach((link, index) => {
+      // Clean and format text for display
+      let displayText = '';
+      if (link.text != null && link.text.trim()) {
+        // Limit length for very long link text
+        let cleanText = link.text.trim();
+        if (cleanText.length > 50) {
+          cleanText = cleanText.substring(0, 47) + '...';
+        }
+        displayText = ` - "${cleanText}"`;
+      }
+
+      appendix += `**link#${index + 1}**: ${link.originalUrl}${displayText}\n\n`;
+    });
+  }
+
+  if (result.images.length > 0) {
+    appendix += '## Images\n\n';
+    result.images.forEach((image, index) => {
+      const displayTitle =
+        image.title != null && image.title.trim()
+          ? ` - ${image.title.trim()}`
+          : '';
+      appendix += `**image#${index + 1}**: ${image.originalUrl}${displayTitle}\n\n`;
+    });
+  }
+
+  if (result.videos.length > 0) {
+    appendix += '## Videos\n\n';
+    result.videos.forEach((video, index) => {
+      const displayTitle =
+        video.title != null && video.title.trim()
+          ? ` - ${video.title.trim()}`
+          : '';
+      appendix += `**video#${index + 1}**: ${video.originalUrl}${displayTitle}\n\n`;
+    });
+  }
+
+  // Add a category breakdown to show what types of links were found
+  const totalRefs =
+    result.links.length + result.images.length + result.videos.length;
+
+  appendix += '## Summary\n\n';
+  appendix += `Total references: **${totalRefs}**\n\n`;
+  appendix += `- Links: ${result.links.length}\n`;
+  appendix += `- Images: ${result.images.length}\n`;
+  appendix += `- Videos: ${result.videos.length}\n`;
+
+  return appendix;
+}
+
+// Using async IIFE to allow for better error handling
+(async (): Promise<void> => {
+  try {
+    await processWikipediaArticle();
+  } catch (error) {
+    console.error('Unhandled error:', error);
+    process.exit(1);
+  }
+})();

package/src/tools/search/tool.ts

@@ -9,9 +9,7 @@ import { formatResultsForLLM } from './format';
 import { createReranker } from './rerankers';
 import { Constants } from '@/common';
 
-const SearchToolSchema = z.object({
-  query: z.string().describe(
-    `
+const DEFAULT_QUERY_DESCRIPTION = `
 GUIDELINES:
 - Start broad, then narrow: Begin with key concepts, then refine with specifics
 - Think like sources: Use terminology experts would use in the field
@@ -29,10 +27,24 @@ TECHNIQUES (combine for power searches):
 - SPECIFIC QUESTIONS: Use who/what/when/where/why/how
 - DOMAIN TERMS: Include technical terminology for specialized topics
 - CONCISE TERMS: Prioritize keywords over sentences
-`.trim()
-  ),
-});
-
+`.trim();
+
+const DEFAULT_COUNTRY_DESCRIPTION = `Country code to localize search results.
+Use standard 2-letter country codes: "us", "uk", "ca", "de", "fr", "jp", "br", etc.
+Provide this when the search should return results specific to a particular country.
+Examples:
+- "us" for United States (default)
+- "de" for Germany
+- "in" for India
+`.trim();
+
+/**
+ * Creates a search tool with a schema that dynamically includes the country field
+ * only when the searchProvider is 'serper'.
+ *
+ * @param config - The search tool configuration
+ * @returns A DynamicStructuredTool with a schema that depends on the searchProvider
+ */
 export const createSearchTool = (
   config: t.SearchToolConfig = {}
 ): DynamicStructuredTool<typeof SearchToolSchema> => {
@@ -53,6 +65,23 @@ export const createSearchTool = (
     onSearchResults: _onSearchResults,
   } = config;
 
+  const querySchema = z.string().describe(DEFAULT_QUERY_DESCRIPTION);
+  const schemaObject: {
+    query: z.ZodString;
+    country?: z.ZodOptional<z.ZodString>;
+  } = {
+    query: querySchema,
+  };
+
+  if (searchProvider === 'serper') {
+    schemaObject.country = z
+      .string()
+      .optional()
+      .describe(DEFAULT_COUNTRY_DESCRIPTION);
+  }
+
+  const SearchToolSchema = z.object(schemaObject);
+
   const searchAPI = createSearchAPI({
     searchProvider,
     serperApiKey,
@@ -88,17 +117,19 @@ export const createSearchTool = (
 
   const search = async ({
     query,
+    country,
     proMode = true,
     maxSources = 5,
     onSearchResults,
   }: {
     query: string;
-    proMode?: boolean;
+    country?: string;
     maxSources?: number;
+    proMode?: boolean;
     onSearchResults?: (sources: t.SearchResult) => void;
   }): Promise<t.SearchResultData> => {
     try {
-      const sources = await searchAPI.getSources(query);
+      const sources = await searchAPI.getSources({ query, country });
       onSearchResults?.(sources);
 
       if (!sources.success) {
@@ -125,9 +156,12 @@ export const createSearchTool = (
   };
 
   return tool<typeof SearchToolSchema>(
-    async ({ query }, runnableConfig) => {
+    async (params, runnableConfig) => {
+      const { query, country: _c } = params;
+      const country = typeof _c === 'string' && _c ? _c : undefined;
       const searchResult = await search({
         query,
+        country,
         onSearchResults: _onSearchResults
           ? (result): void => {
             _onSearchResults(result, runnableConfig);
@@ -135,17 +169,22 @@ export const createSearchTool = (
           : undefined,
       });
       const turn = runnableConfig.toolCall?.turn ?? 0;
-      const output = formatResultsForLLM(turn, searchResult);
-      return [output, { [Constants.WEB_SEARCH]: { turn, ...searchResult } }];
+      const { output, references } = formatResultsForLLM(turn, searchResult);
+      return [
+        output,
+        { [Constants.WEB_SEARCH]: { turn, ...searchResult, references } },
+      ];
     },
     {
       name: Constants.WEB_SEARCH,
       description: `
-Real-time search. Results have required unique citation anchors.
+Real-time search. Results have required citation anchors.
+
+Note: Use ONCE per reply unless instructed otherwise.
 
 Anchors:
-- \\ue202turnXsearchY (web), \\ue202turnXnewsY (news), \\ue202turnXimageY (image)
-- X = turn, Y = item number
+- \\ue202turnXtypeY
+- X = turn idx, type = 'search' | 'news' | 'image' | 'ref', Y = item idx
 
 Special Markers:
 - \\ue203...\\ue204 — highlight start/end of cited text (for Standalone or Group citations)