chat-agent-toolkit 1.2.1

package/src/tools/qwksearch-api-tools.ts

@@ -0,0 +1,327 @@
+/**
+ * @fileoverview Specialized tools for AI agents to interact with the QwkSearch API.
+ * Provides web search, content extraction, and AI response generation.
+ */
+
+// @ts-nocheck
+import { z } from "zod";
+import * as QwkSearch from 'qwksearch-api-client'; // Update this path to match your QwkSearch module location
+
+// Configuration for QwkSearch API
+const QWKSEARCH_CONFIG = {
+  baseURL: typeof process !== "undefined" && process?.env.QWKSEARCH_URL || 'https://app.qwksearch.com/api',
+  apiKey: typeof process !== "undefined" && process?.env.QWKSEARCH_API_KEY || null,
+};
+
+/**
+ * List of tools available for agent usage, including their schemas and implementation.
+ */
+export const AGENT_TOOLS = [
+
+  {
+    name: "web_search",
+    description:
+      "Search the web for information on any topic using QwkSearch API. Input: search query string and optional category. Returns relevant search results with titles, descriptions, and URLs from 100+ sources via SearXNG metasearch engine.",
+    schema: z.object({
+      query: z.string(),
+      category: z.enum(["general", "news", "videos", "images", "science", "files", "it"]).optional().default("general"),
+      recency: z.enum(["none", "day", "week", "month", "year"]).optional().default("none"),
+      page: z.number().optional().default(1),
+      language: z.string().optional().default("en-US"),
+      public: z.boolean().optional().default(false),
+      timeout: z.number().optional().default(10),
+      baseURL: z.string().optional(),
+      apiKey: z.string().optional()
+    }),
+    func: async ({ query, category = "general", recency = "none", page = 1, language = "en-US", public: isPublic = false, timeout = 10, baseURL, apiKey }) => {
+      try {
+        // Use provided config or fall back to environment/defaults
+        const baseUrl = baseURL || QWKSEARCH_CONFIG.baseURL;
+        const headers = apiKey || QWKSEARCH_CONFIG.apiKey ? { 'x-api-key': apiKey || QWKSEARCH_CONFIG.apiKey } : undefined;
+
+        const result = await QwkSearch.searchWeb({
+          query: {
+            q: query,
+            cat: category,
+            recency: recency,
+            page: page,
+            lang: language,
+            public: isPublic,
+            timeout: timeout
+          },
+          baseUrl: baseUrl,
+          ...(headers && { headers })
+        });
+
+        if (!result.data || !result.data.results || result.data.results.length === 0) {
+          return `No search results found for "${query}". Please try a different search term.`;
+        }
+
+        let resultText = `Web search results for "${query}" (${category} category):\n\n`;
+
+        result.data.results.forEach((searchResult, index) => {
+          resultText += `${index + 1}. ${searchResult.title}\n`;
+          resultText += `   URL: ${searchResult.url}\n`;
+          if (searchResult.domain) {
+            resultText += `   Domain: ${searchResult.domain}\n`;
+          }
+          if (searchResult.snippet) {
+            resultText += `   Description: ${searchResult.snippet}\n`;
+          }
+          if (searchResult.engines && searchResult.engines.length > 0) {
+            resultText += `   Sources: ${searchResult.engines.join(", ")}\n`;
+          }
+          resultText += `\n`;
+        });
+
+        resultText += `Found ${result.data.results.length} results from multiple search engines. This is the complete search information.`;
+
+        return resultText;
+      } catch (error) {
+        return `Unable to perform web search for "${query}". Error: ${error.message}`;
+      }
+    },
+  },
+  {
+    name: "extract_page",
+    description:
+      "Extract and summarize content from a web page using QwkSearch API. Supports articles, PDFs, and YouTube videos. Uses Mozilla Readability and Postlight Mercury algorithms with 100+ custom adapters for major sites. Input: URL of the page to extract. Returns structured content with citation information.",
+    schema: z.object({
+      url: z.string().url(),
+      images: z.boolean().optional().default(true),
+      links: z.boolean().optional().default(true),
+      formatting: z.boolean().optional().default(true),
+      absoluteURLs: z.boolean().optional().default(true),
+      timeout: z.number().min(1).max(30).optional().default(10),
+      baseURL: z.string().optional(),
+      apiKey: z.string().optional()
+    }),
+    func: async ({ url, images = true, links = true, formatting = true, absoluteURLs = true, timeout = 10, baseURL, apiKey }) => {
+      try {
+        // Use provided config or fall back to environment/defaults
+        const baseUrl = baseURL || QWKSEARCH_CONFIG.baseURL;
+        const headers = apiKey || QWKSEARCH_CONFIG.apiKey ? { 'x-api-key': apiKey || QWKSEARCH_CONFIG.apiKey } : undefined;
+
+        const result = await QwkSearch.extractContent({
+          query: {
+            url: url,
+            images: images,
+            links: links,
+            formatting: formatting,
+            absoluteURLs: absoluteURLs,
+            timeout: timeout
+          },
+          baseUrl: baseUrl,
+          ...(headers && { headers })
+        });
+
+        if (!result.data) {
+          return `No content could be extracted from "${url}". Please check the URL and try again.`;
+        }
+
+        const data = result.data;
+
+        let resultText = `Content extracted from: ${data.url || url}\n\n`;
+
+        if (data.title) {
+          resultText += `Title: ${data.title}\n\n`;
+        }
+
+        if (data.author) {
+          resultText += `Author: ${data.author}\n`;
+          if (data.author_cite) {
+            resultText += `Author (Citation Format): ${data.author_cite}\n`;
+          }
+          if (data.author_type) {
+            resultText += `Author Type: ${data.author_type}\n`;
+          }
+        }
+
+        if (data.date) {
+          resultText += `Publication Date: ${data.date}\n`;
+        }
+
+        if (data.source) {
+          resultText += `Source: ${data.source}\n`;
+        }
+
+        if (data.word_count) {
+          resultText += `Word Count: ${data.word_count}\n`;
+        }
+
+        if (data.cite) {
+          resultText += `\nCitation (APA Format): ${data.cite}\n`;
+        }
+
+        if (data.html) {
+          resultText += `\nContent:\n${data.html}\n\n`;
+        }
+
+        resultText += `This is the complete page extraction information.`;
+
+        return resultText;
+      } catch (error) {
+        return `Unable to extract content from "${url}". Error: ${error.message}`;
+      }
+    },
+  },
+  {
+    name: "generate_ai_response",
+    description:
+      "Generate AI language model responses using QwkSearch API with various agent templates. Supports multiple providers (Groq, OpenAI, Anthropic, etc.) and agent types for different tasks like summarization, question answering, and content generation.",
+    schema: z.object({
+      provider: z.enum(["groq", "openai", "anthropic", "together", "xai", "google", "perplexity", "cloudflare"]),
+      key: z.string().optional(),
+      agent: z.enum([
+        "question",
+        "summarize-bullets",
+        "summarize",
+        "suggest-followups",
+        "answer-cite-sources",
+        "query-resolution",
+        "knowledge-graph-nodes",
+        "summary-longtext"
+      ]).optional().default("question"),
+      model: z.string().optional().default("meta-llama/llama-4-maverick-17b-128e-instruct"),
+      temperature: z.number().min(0).max(2).optional().default(0.7),
+      html: z.boolean().optional().default(true),
+      query: z.string().optional(),
+      chat_history: z.string().optional(),
+      article: z.string().optional(),
+      baseURL: z.string().optional(),
+      apiKey: z.string().optional()
+    }),
+    func: async ({ provider, key, agent = "question", model = "meta-llama/llama-4-maverick-17b-128e-instruct", temperature = 0.7, html = true, query, chat_history, article, baseURL, apiKey }) => {
+      try {
+        // Use provided config or fall back to environment/defaults
+        const baseUrl = baseURL || QWKSEARCH_CONFIG.baseURL;
+        const headers = apiKey || QWKSEARCH_CONFIG.apiKey ? { 'x-api-key': apiKey || QWKSEARCH_CONFIG.apiKey } : undefined;
+
+        const requestBody = {
+          agent,
+          provider,
+          model,
+          html,
+          temperature
+        };
+
+        if (key) {
+          requestBody.key = key;
+        }
+
+        if (query) {
+          requestBody.query = query;
+        }
+
+        if (chat_history) {
+          requestBody.chat_history = chat_history;
+        }
+
+        if (article) {
+          requestBody.article = article;
+        }
+
+        const result = await QwkSearch.writeLanguage({
+          body: requestBody,
+          baseUrl: baseUrl,
+          ...(headers && { headers })
+        });
+
+        if (!result.data) {
+          return `No response generated. Please check your input parameters and try again.`;
+        }
+
+        let resultText = `AI Response (${agent} agent, ${provider} provider):\n\n`;
+
+        if (result.data.content) {
+          resultText += result.data.content;
+        }
+
+        if (result.data.extract) {
+          resultText += `\n\nStructured Extract:\n${JSON.stringify(result.data.extract, null, 2)}`;
+        }
+
+        resultText += `\n\nThis is the complete AI-generated response.`;
+
+        return resultText;
+      } catch (error) {
+        return `Unable to generate AI response. Error: ${error.message}`;
+      }
+    },
+  },
+  {
+    name: "render_page_with_javascript",
+    description:
+      "Render a web page with JavaScript execution using Cloudflare Browser Rendering. Use this for JavaScript-heavy sites (SPAs, React apps), pages behind bot protection (Cloudflare, reCAPTCHA), or when extract_page fails. Returns fully rendered HTML with JavaScript executed. Slower but more complete than extract_page.",
+    schema: z.object({
+      url: z.string().url(),
+      blockImages: z.boolean().optional().default(true),
+      wait: z.number().min(0).max(10000).optional().default(0),
+      timeout: z.number().min(1000).max(60000).optional().default(30000),
+      waitUntil: z.enum(["domcontentloaded", "load", "networkidle0", "networkidle2"]).optional().default("networkidle2"),
+      bypassCaptcha: z.boolean().optional().default(true),
+      sessionId: z.string().optional().default("default"),
+      format: z.enum(["html", "json"]).optional().default("html"),
+      scraperUrl: z.string().optional(),
+      scraperApiKey: z.string().optional()
+    }),
+    func: async ({ url, blockImages = true, wait = 0, timeout = 30000, waitUntil = "networkidle2", bypassCaptcha = true, sessionId = "default", format = "html", scraperUrl, scraperApiKey }) => {
+      try {
+        const scraperEndpoint = scraperUrl || (typeof process !== "undefined" && process?.env?.SCRAPER_URL) || 'https://scraper.qwksearch.workers.dev';
+        const apiKey = scraperApiKey || (typeof process !== "undefined" && process?.env?.SCRAPER_API_KEY);
+
+        const requestUrl = new URL('/api/render', scraperEndpoint);
+
+        const body = {
+          url,
+          blockImages,
+          wait,
+          timeout,
+          waitUntil,
+          bypassCaptcha,
+          sessionId,
+          format
+        };
+
+        const headers: Record<string, string> = {
+          'Content-Type': 'application/json',
+        };
+
+        if (apiKey) {
+          headers['Authorization'] = `Bearer ${apiKey}`;
+        }
+
+        const response = await fetch(requestUrl.toString(), {
+          method: 'POST',
+          headers,
+          body: JSON.stringify(body),
+        });
+
+        if (!response.ok) {
+          const errorText = await response.text();
+          return `Failed to render page: ${errorText}`;
+        }
+
+        if (format === 'json') {
+          const data = await response.json();
+          let resultText = `Page rendered successfully: ${data.url}\n\n`;
+          if (data.title) {
+            resultText += `Title: ${data.title}\n`;
+          }
+          resultText += `Load Time: ${data.loadTime}ms\n`;
+          if (data.challengeBypassed) {
+            resultText += `Challenge Bypassed: Yes (${data.retryCount} retries)\n`;
+          }
+          resultText += `\nRendered HTML Content:\n${data.html}\n\n`;
+          resultText += `This is the complete rendered page content.`;
+          return resultText;
+        }
+
+        const html = await response.text();
+        return `Page rendered successfully.\n\nHTML Content:\n${html}`;
+      } catch (error) {
+        return `Unable to render page "${url}". Error: ${error.message}`;
+      }
+    },
+  },
+];

package/src/tools/search/doc-utils.ts

@@ -0,0 +1,75 @@
+/**
+ * @module research/search/doc-utils
+ * @description Document utilities: fallback docs, reranking, and formatting.
+ */
+import type { Document } from "./document";
+import path from "node:path";
+import fs from "node:fs";
+
+export function buildFallbackDocs(query: string): Document[] {
+  const trimmedQuery = (query || "").trim();
+  const searchQuery = trimmedQuery.length > 0 ? trimmedQuery : "web search";
+  const fallbackUrl = `https://www.google.com/search?q=${encodeURIComponent(searchQuery)}`;
+
+  return [
+    {
+      pageContent:
+        "No indexed sources were returned by the configured search providers for this query.",
+      metadata: {
+        title: `Search results for: ${searchQuery}`,
+        url: fallbackUrl,
+        source: "Google Search",
+      },
+    },
+  ];
+}
+
+export function normalizeSourcesOutput(output: unknown, query: string): Document[] {
+  if (Array.isArray(output) && output.length > 0) {
+    return output as Document[];
+  }
+  return buildFallbackDocs(query);
+}
+
+export async function rerankDocs(
+  query: string,
+  docs: Document[],
+  fileIds: string[],
+  optimizationMode: "speed" | "balanced" | "quality",
+): Promise<Document[]> {
+  if (docs.length === 0 && fileIds.length === 0) {
+    return docs;
+  }
+
+  const filesData = fileIds.map((file) => {
+    const filePath = path.join(process.cwd(), "uploads", file);
+    const contentPath = filePath + "-extracted.json";
+    const content = JSON.parse(fs.readFileSync(contentPath, "utf8"));
+    return { fileName: content.title, content: content.content };
+  });
+
+  if (query.toLocaleLowerCase() === "summarize") {
+    return docs.slice(0, 15);
+  }
+
+  const docsWithContent = docs.filter(
+    (doc) => doc.pageContent && doc.pageContent.length > 0,
+  );
+
+  const fileDocs: Document[] = filesData.map((fileData) => ({
+    pageContent: fileData.content,
+    metadata: { title: fileData.fileName, url: "File" },
+  }));
+
+  // Combine file docs with web results, cap at 15
+  return [...fileDocs, ...docsWithContent].slice(0, 15);
+}
+
+export function processDocs(docs: Document[]): string {
+  return docs
+    .map(
+      (_, index) =>
+        `${index + 1}. ${docs[index].metadata.title} ${docs[index].pageContent}`,
+    )
+    .join("\n");
+}

package/src/tools/search/document.ts

@@ -0,0 +1,47 @@
+/**
+ * @module research/search/document
+ * @description Minimal document shape shared across the search pipeline.
+ * A document is a chunk of page content plus citation metadata (title, url, source, etc.).
+ */
+
+export interface Document<
+  Metadata extends Record<string, any> = Record<string, any>,
+> {
+  pageContent: string;
+  metadata: Metadata;
+}
+
+/**
+ * Splits text into overlapping chunks on whitespace boundaries.
+ * Default chunkSize 1000, chunkOverlap 200.
+ */
+export function splitTextIntoChunks(
+  text: string,
+  chunkSize = 1000,
+  chunkOverlap = 200,
+): string[] {
+  const trimmed = (text || "").trim();
+  if (trimmed.length <= chunkSize) {
+    return trimmed.length > 0 ? [trimmed] : [];
+  }
+
+  const chunks: string[] = [];
+  let start = 0;
+
+  while (start < trimmed.length) {
+    let end = Math.min(start + chunkSize, trimmed.length);
+
+    // Break on the last whitespace inside the window so words stay intact
+    if (end < trimmed.length) {
+      const lastSpace = trimmed.lastIndexOf(" ", end);
+      if (lastSpace > start) end = lastSpace;
+    }
+
+    chunks.push(trimmed.slice(start, end).trim());
+
+    if (end >= trimmed.length) break;
+    start = Math.max(end - chunkOverlap, start + 1);
+  }
+
+  return chunks.filter((chunk) => chunk.length > 0);
+}

package/src/tools/search/index.ts

@@ -0,0 +1,13 @@
+/**
+ * @module agent-toolkit/tools/search
+ * @description Meta search agent tools for AI-powered web search and research
+ */
+
+export { default as MetaSearchAgent } from "./metaSearchAgent";
+export type { MetaSearchAgentType, Config, ChatTurnMessage, SearchingEvent, FewShotExample } from "./meta-search-types";
+export { searchHandlers, createSearchHandlers } from "./search-handlers";
+export { default as generateSuggestions } from "./suggestionGeneratorAgent";
+export { groupAndSummarizeDocs } from "./link-summarizer";
+export type { Document } from "./document";
+export { splitTextIntoChunks } from "./document";
+export { buildFallbackDocs, rerankDocs, processDocs, normalizeSourcesOutput } from "./doc-utils";

package/src/tools/search/link-summarizer.ts

@@ -0,0 +1,112 @@
+/**
+ * @module research/search/link-summarizer
+ * @description Groups link documents by URL then summarizes each group with an LLM.
+ */
+import { generateText, type LanguageModel } from "ai";
+import type { Document } from "./document";
+import { buildFallbackDocs } from "./doc-utils";
+
+const SUMMARIZER_PROMPT = `You are a web search summarizer, tasked with summarizing a piece of text retrieved from a web search. Your job is to summarize the
+text into a detailed, 2-4 paragraph explanation that captures the main ideas and provides a comprehensive answer to the query.
+If the query is "summarize", you should provide a detailed summary of the text. If the query is a specific question, you should answer it in the summary.
+
+- **Journalistic tone**: The summary should sound professional and journalistic, not too casual or vague.
+- **Thorough and detailed**: Ensure that every key point from the text is captured and that the summary directly answers the query.
+- **Not too lengthy, but detailed**: The summary should be informative but not excessively long. Focus on providing detailed information in a concise format.
+
+The text will be shared inside the \`text\` XML tag, and the query inside the \`query\` XML tag.
+
+<example>
+1. \`<text>
+Docker is a set of platform-as-a-service products that use OS-level virtualization to deliver software in packages called containers.
+It was first released in 2013 and is developed by Docker, Inc. Docker is designed to make it easier to create, deploy, and run applications
+by using containers.
+</text>
+
+<query>
+What is Docker and how does it work?
+</query>
+
+Response:
+Docker is a revolutionary platform-as-a-service product developed by Docker, Inc., that uses container technology to make application
+deployment more efficient. It allows developers to package their software with all necessary dependencies, making it easier to run in
+any environment. Released in 2013, Docker has transformed the way applications are built, deployed, and managed.
+\`
+2. \`<text>
+The theory of relativity, or simply relativity, encompasses two interrelated theories of Albert Einstein: special relativity and general
+relativity. However, the word "relativity" is sometimes used in reference to Galilean invariance. The term "theory of relativity" was based
+on the expression "relative theory" used by Max Planck in 1906. The theory of relativity usually encompasses two interrelated theories by
+Albert Einstein: special relativity and general relativity. Special relativity applies to all physical phenomena in the absence of gravity.
+General relativity explains the law of gravitation and its relation to other forces of nature. It applies to the cosmological and astrophysical
+realm, including astronomy.
+</text>
+
+<query>
+summarize
+</query>
+
+Response:
+The theory of relativity, developed by Albert Einstein, encompasses two main theories: special relativity and general relativity. Special
+relativity applies to all physical phenomena in the absence of gravity, while general relativity explains the law of gravitation and its
+relation to other forces of nature. The theory of relativity is based on the concept of "relative theory," as introduced by Max Planck in
+1906. It is a fundamental theory in physics that has revolutionized our understanding of the universe.
+\`
+</example>
+
+Everything below is the actual data you will be working with. Good luck!
+
+<query>
+{question}
+</query>
+
+<text>
+{content}
+</text>
+
+Make sure to answer the query in the summary.`;
+
+/**
+ * Groups fetched link documents by URL (max 10 chunks per URL), then
+ * summarizes each group with the LLM in parallel.
+ */
+export async function groupAndSummarizeDocs(
+  llm: LanguageModel,
+  linkDocs: Document[],
+  question: string,
+): Promise<Document[]> {
+  // Group chunks by URL, capping at 10 chunks each
+  const docGroups: Document[] = [];
+
+  for (const doc of linkDocs) {
+    const existing = docGroups.find(
+      (d) => d.metadata.url === doc.metadata.url && d.metadata.totalDocs < 10,
+    );
+
+    if (!existing) {
+      docGroups.push({ ...doc, metadata: { ...doc.metadata, totalDocs: 1 } });
+    } else {
+      existing.pageContent += `\n\n${doc.pageContent}`;
+      existing.metadata.totalDocs += 1;
+    }
+  }
+
+  // Summarize each group in parallel
+  const docs: Document[] = [];
+
+  await Promise.all(
+    docGroups.map(async (doc) => {
+      const prompt = SUMMARIZER_PROMPT.replace("{question}", question).replace(
+        "{content}",
+        doc.pageContent,
+      );
+      const { text } = await generateText({ model: llm, prompt });
+
+      docs.push({
+        pageContent: text,
+        metadata: { title: doc.metadata.title, url: doc.metadata.url },
+      });
+    }),
+  );
+
+  return docs.length > 0 ? docs : buildFallbackDocs(question);
+}

package/src/tools/search/meta-search-types.ts

@@ -0,0 +1,62 @@
+/**
+ * @module research/search/meta-search-types
+ * @description Shared types for the MetaSearchAgent.
+ */
+import type { LanguageModel } from "ai";
+import type EventEmitter from "events";
+
+/** A single conversation turn, matching the Vercel AI SDK message shape. */
+export interface ChatTurnMessage {
+  role: "user" | "assistant" | "system";
+  content: string;
+}
+
+/** A `[role, content]` tuple used for few-shot prompt examples. */
+export type FewShotExample = [role: "user" | "assistant", content: string];
+
+export interface MetaSearchAgentType {
+  searchAndAnswer: (
+    message: string,
+    history: ChatTurnMessage[],
+    llm: LanguageModel,
+    optimizationMode: "speed" | "balanced" | "quality",
+    fileIds: string[],
+    systemInstructions: string,
+    category?: string,
+    sourceExtractionEnabled?: boolean,
+    thinkingTimeLimit?: number,
+  ) => Promise<EventEmitter>;
+}
+
+/** Emitted on the EventEmitter data channel to report live search progress. */
+export interface SearchingEvent {
+  query: string;
+  /** Display label shown in the UI (e.g. "Academic · max 10"). */
+  category?: string;
+  status: "running" | "done";
+}
+
+export interface Config {
+  searchWeb: boolean;
+  rerank: boolean;
+  rerankThreshold: number;
+  queryGeneratorPrompt: string;
+  queryGeneratorFewShots: FewShotExample[];
+  responsePrompt: string;
+  activeEngines: string[];
+  /** Optional: Function to fetch documents from URLs */
+  getDocumentsFromLinks?: (options: { links: string[] }) => Promise<any[]>;
+  /** Optional: Function to search via SearXNG */
+  searchSearxng?: (query: string, options: any) => Promise<{ results: any[]; suggestions: string[] }>;
+  /** Optional: Function to search via Tavily */
+  searchTavily?: (query: string, options: any) => Promise<{ results: any[]; suggestions: string[] }>;
+  /** Optional: Function to check if Tavily is configured */
+  isTavilyConfigured?: () => boolean;
+  /** Optional: Function to scrape a URL */
+  scrapeURL?: (url: string, options: any) => Promise<string>;
+}
+
+export type BasicChainInput = {
+  chat_history: ChatTurnMessage[];
+  query: string;
+};