chat-agent-toolkit 1.2.48 → 1.2.50

package/src/mastra/rag.ts

@@ -0,0 +1,188 @@
+/**
+ * @fileoverview Mastra RAG (Retrieval-Augmented Generation)
+ *
+ * Document processing, chunking, embedding, and vector-store retrieval.
+ * Uses Vercel AI SDK embeddings with pluggable vector backends.
+ */
+
+import { embed, embedMany } from "ai";
+
+export interface RAGConfig {
+  embeddingModel: any;
+  chunkSize?: number;
+  chunkOverlap?: number;
+  topK?: number;
+}
+
+export interface RAGDocument {
+  id: string;
+  content: string;
+  metadata?: Record<string, any>;
+}
+
+export interface RAGChunk {
+  id: string;
+  documentId: string;
+  text: string;
+  embedding: number[];
+  metadata?: Record<string, any>;
+}
+
+export interface RAGRetrievalResult {
+  chunk: RAGChunk;
+  score: number;
+}
+
+/**
+ * In-memory RAG pipeline with document chunking, embedding, and retrieval.
+ * For production, swap the vector store with Pinecone, Qdrant, or pgvector.
+ *
+ * @example
+ * ```ts
+ * import { MastraRAG } from "chat-agent-toolkit/mastra";
+ * import { openai } from "@ai-sdk/openai";
+ *
+ * const rag = new MastraRAG({
+ *   embeddingModel: openai.embedding("text-embedding-3-small"),
+ *   chunkSize: 512,
+ *   topK: 5,
+ * });
+ *
+ * await rag.ingest([
+ *   { id: "doc1", content: "Long document text..." },
+ * ]);
+ *
+ * const results = await rag.retrieve("What is the main topic?");
+ * ```
+ */
+export class MastraRAG {
+  private config: Required<RAGConfig>;
+  private chunks: RAGChunk[] = [];
+
+  constructor(config: RAGConfig) {
+    this.config = {
+      embeddingModel: config.embeddingModel,
+      chunkSize: config.chunkSize ?? 512,
+      chunkOverlap: config.chunkOverlap ?? 64,
+      topK: config.topK ?? 5,
+    };
+  }
+
+  /**
+   * Split text into overlapping chunks of configured size.
+   */
+  private splitIntoChunks(text: string): string[] {
+    const chunks: string[] = [];
+    const words = text.split(/\s+/);
+    const { chunkSize, chunkOverlap } = this.config;
+
+    let start = 0;
+    while (start < words.length) {
+      const end = Math.min(start + chunkSize, words.length);
+      chunks.push(words.slice(start, end).join(" "));
+      start += chunkSize - chunkOverlap;
+    }
+
+    return chunks;
+  }
+
+  /**
+   * Ingest documents: chunk, embed, and store.
+   */
+  async ingest(documents: RAGDocument[]): Promise<number> {
+    let totalChunks = 0;
+
+    for (const doc of documents) {
+      const textChunks = this.splitIntoChunks(doc.content);
+
+      const { embeddings } = await embedMany({
+        model: this.config.embeddingModel,
+        values: textChunks,
+      });
+
+      for (let i = 0; i < textChunks.length; i++) {
+        this.chunks.push({
+          id: `${doc.id}_chunk_${i}`,
+          documentId: doc.id,
+          text: textChunks[i],
+          embedding: embeddings[i],
+          metadata: doc.metadata,
+        });
+      }
+
+      totalChunks += textChunks.length;
+    }
+
+    return totalChunks;
+  }
+
+  /**
+   * Retrieve relevant chunks for a query using cosine similarity.
+   */
+  async retrieve(query: string): Promise<RAGRetrievalResult[]> {
+    if (this.chunks.length === 0) return [];
+
+    const { embedding } = await embed({
+      model: this.config.embeddingModel,
+      value: query,
+    });
+
+    const scored = this.chunks.map((chunk) => ({
+      chunk,
+      score: cosineSimilarity(embedding, chunk.embedding),
+    }));
+
+    scored.sort((a, b) => b.score - a.score);
+
+    return scored.slice(0, this.config.topK);
+  }
+
+  /**
+   * Retrieve and format context string for LLM consumption.
+   */
+  async getContext(query: string): Promise<string> {
+    const results = await this.retrieve(query);
+    if (results.length === 0) return "";
+
+    return results
+      .map((r, i) => `[${i + 1}] (score: ${r.score.toFixed(3)}) ${r.chunk.text}`)
+      .join("\n\n");
+  }
+
+  /**
+   * Get total chunk count in the store.
+   */
+  get size(): number {
+    return this.chunks.length;
+  }
+
+  /**
+   * Clear all stored chunks.
+   */
+  clear(): void {
+    this.chunks = [];
+  }
+}
+
+/**
+ * Factory for creating a configured RAG pipeline.
+ */
+export function createRAGPipeline(config: RAGConfig): MastraRAG {
+  return new MastraRAG(config);
+}
+
+function cosineSimilarity(a: number[], b: number[]): number {
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+
+  const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+  if (denominator === 0) return 0;
+  return dotProduct / denominator;
+}

package/src/mastra/telemetry.ts

@@ -0,0 +1,71 @@
+/**
+ * @fileoverview Mastra Telemetry & Instance Factory
+ *
+ * Creates configured Mastra instances with OpenTelemetry-compatible
+ * tracing, logging, and agent registration.
+ */
+
+import { Mastra } from "@mastra/core";
+
+export interface TelemetryConfig {
+  serviceName: string;
+  enabled?: boolean;
+  sampling?: number;
+  exporterUrl?: string;
+}
+
+export interface MastraInstanceConfig {
+  agents?: Record<string, any>;
+  workflows?: Record<string, any>;
+  telemetry?: TelemetryConfig;
+  storage?: any;
+}
+
+/**
+ * Create a fully configured Mastra instance with agents, workflows,
+ * and telemetry. Central entry point for Mastra framework setup.
+ *
+ * @example
+ * ```ts
+ * import { createMastraInstance } from "chat-agent-toolkit/mastra";
+ *
+ * const mastra = createMastraInstance({
+ *   agents: { assistant, researcher },
+ *   workflows: { researchWorkflow },
+ *   telemetry: {
+ *     serviceName: "my-app",
+ *     enabled: true,
+ *   },
+ * });
+ *
+ * const agent = mastra.getAgent("assistant");
+ * const result = await agent.generate("Hello");
+ * ```
+ */
+export function createMastraInstance(config: MastraInstanceConfig): Mastra {
+  const mastraConfig: any = {};
+
+  if (config.agents) {
+    mastraConfig.agents = config.agents;
+  }
+
+  if (config.workflows) {
+    mastraConfig.workflows = config.workflows;
+  }
+
+  if (config.telemetry) {
+    mastraConfig.telemetry = {
+      serviceName: config.telemetry.serviceName,
+      enabled: config.telemetry.enabled ?? true,
+      sampling: {
+        default: config.telemetry.sampling ?? 1.0,
+      },
+    };
+  }
+
+  if (config.storage) {
+    mastraConfig.storage = config.storage;
+  }
+
+  return new Mastra(mastraConfig);
+}

package/src/mastra/workflows.ts

@@ -0,0 +1,155 @@
+/**
+ * @fileoverview Mastra Workflow Builder
+ *
+ * Graph-based deterministic workflows with typed steps, chaining,
+ * branching, and parallel execution support.
+ */
+
+import { createWorkflow, createStep } from "@mastra/core/workflows";
+import { z } from "zod";
+
+export interface WorkflowStepConfig<TInput = any, TOutput = any> {
+  id: string;
+  inputSchema: z.ZodType<TInput>;
+  outputSchema: z.ZodType<TOutput>;
+  execute: (params: { inputData: TInput }) => Promise<TOutput>;
+}
+
+/**
+ * Create a typed workflow step.
+ *
+ * @example
+ * ```ts
+ * const fetchStep = createWorkflowStep({
+ *   id: "fetch-sources",
+ *   inputSchema: z.object({ query: z.string() }),
+ *   outputSchema: z.object({ sources: z.array(z.string()) }),
+ *   execute: async ({ inputData }) => {
+ *     const sources = await searchWeb(inputData.query);
+ *     return { sources };
+ *   },
+ * });
+ * ```
+ */
+export function createWorkflowStep<TInput, TOutput>(
+  config: WorkflowStepConfig<TInput, TOutput>
+) {
+  return createStep({
+    id: config.id,
+    inputSchema: config.inputSchema,
+    outputSchema: config.outputSchema,
+    execute: config.execute,
+  });
+}
+
+/**
+ * Create a research workflow: fetch → analyze → summarize.
+ * Pre-built pipeline for common research tasks.
+ *
+ * @example
+ * ```ts
+ * const workflow = createResearchWorkflow({
+ *   fetchFn: async (topic) => fetchSources(topic),
+ *   analyzeFn: async (sources) => analyzeContent(sources),
+ *   summarizeFn: async (analysis) => generateSummary(analysis),
+ * });
+ *
+ * const run = await workflow.createRunAsync();
+ * const result = await run.start({ inputData: { topic: "quantum computing" } });
+ * ```
+ */
+export function createResearchWorkflow(handlers: {
+  fetchFn: (topic: string) => Promise<string[]>;
+  analyzeFn: (sources: string[]) => Promise<string>;
+  summarizeFn: (analysis: string) => Promise<string>;
+}) {
+  const fetchStep = createStep({
+    id: "fetch-sources",
+    inputSchema: z.object({ topic: z.string() }),
+    outputSchema: z.object({ sources: z.array(z.string()) }),
+    execute: async ({ inputData }) => {
+      const sources = await handlers.fetchFn(inputData.topic);
+      return { sources };
+    },
+  });
+
+  const analyzeStep = createStep({
+    id: "analyze-content",
+    inputSchema: z.object({ sources: z.array(z.string()) }),
+    outputSchema: z.object({ analysis: z.string() }),
+    execute: async ({ inputData }) => {
+      const analysis = await handlers.analyzeFn(inputData.sources);
+      return { analysis };
+    },
+  });
+
+  const summarizeStep = createStep({
+    id: "summarize",
+    inputSchema: z.object({ analysis: z.string() }),
+    outputSchema: z.object({ summary: z.string() }),
+    execute: async ({ inputData }) => {
+      const summary = await handlers.summarizeFn(inputData.analysis);
+      return { summary };
+    },
+  });
+
+  return createWorkflow({
+    id: "research-workflow",
+    inputSchema: z.object({ topic: z.string() }),
+    outputSchema: z.object({ summary: z.string() }),
+  })
+    .then(fetchStep)
+    .then(analyzeStep)
+    .then(summarizeStep)
+    .commit();
+}
+
+/**
+ * Create a RAG workflow: chunk → embed → retrieve → generate.
+ * End-to-end retrieval-augmented generation pipeline.
+ *
+ * @example
+ * ```ts
+ * const workflow = createRAGWorkflow({
+ *   chunkFn: async (doc) => splitIntoChunks(doc),
+ *   embedFn: async (chunks) => embedChunks(chunks),
+ *   retrieveFn: async (query) => findRelevant(query),
+ *   generateFn: async (context, query) => llmGenerate(context, query),
+ * });
+ * ```
+ */
+export function createRAGWorkflow(handlers: {
+  chunkFn: (document: string) => Promise<string[]>;
+  embedFn: (chunks: string[]) => Promise<number[][]>;
+  retrieveFn: (query: string) => Promise<string[]>;
+  generateFn: (context: string[], query: string) => Promise<string>;
+}) {
+  const retrieveStep = createStep({
+    id: "retrieve",
+    inputSchema: z.object({ query: z.string() }),
+    outputSchema: z.object({ context: z.array(z.string()), query: z.string() }),
+    execute: async ({ inputData }) => {
+      const context = await handlers.retrieveFn(inputData.query);
+      return { context, query: inputData.query };
+    },
+  });
+
+  const generateStep = createStep({
+    id: "generate",
+    inputSchema: z.object({ context: z.array(z.string()), query: z.string() }),
+    outputSchema: z.object({ answer: z.string() }),
+    execute: async ({ inputData }) => {
+      const answer = await handlers.generateFn(inputData.context, inputData.query);
+      return { answer };
+    },
+  });
+
+  return createWorkflow({
+    id: "rag-workflow",
+    inputSchema: z.object({ query: z.string() }),
+    outputSchema: z.object({ answer: z.string() }),
+  })
+    .then(retrieveStep)
+    .then(generateStep)
+    .commit();
+}

package/src/tools/search/search-handlers.ts

@@ -3,8 +3,7 @@
  * @description Research library module.
  *
  * Note: To use these search handlers, you need to provide search functions
- * (searchSearxng, searchTavily, etc.) from extract-webpage package.
- * See extract-webpage/src/search/index.ts for an example.
+ * (searchSearxng, searchTavily, etc.) from search-web-api package.
  */
 import MetaSearchAgent from "./metaSearchAgent";
 import {

package/src/utils/markdown-to-html.ts

@@ -1,11 +1,14 @@
 /**
  * @module research/agents/markdown-to-html
- * @description Converts Markdown to HTML with Highlight.js syntax highlighting.
+ * @description Converts Markdown to HTML with Prism.js syntax highlighting.
  */
 import { marked } from "marked";
-import hljs from "highlight.js";
+import Prism from "prismjs";
+import loadLanguages from "prismjs/components/index.js";
 import { encode, decode } from "html-entities";
 
+loadLanguages();
+
 // Inline onclick: self-contained per button, works in dangerouslySetInnerHTML contexts
 const COPY_ONCLICK = [
   "(function(b){",
@@ -28,22 +31,22 @@ marked.use({
       let highlighted: string;
 
       try {
-        if (language && hljs.getLanguage(language)) {
-          highlighted = hljs.highlight(text, { language }).value;
+        if (language && Prism.languages[language]) {
+          highlighted = Prism.highlight(text, Prism.languages[language], language);
         } else {
-          highlighted = hljs.highlightAuto(text).value;
+          highlighted = encode(text);
         }
       } catch (e) {
         highlighted = encode(text);
       }
 
-      return `<figure class="code-block"><button class="code-copy-btn" type="button" onclick="${COPY_ONCLICK}">Copy</button><pre><code class="hljs language-${language}">${highlighted}</code></pre></figure>`;
+      return `<figure class="code-block"><button class="code-copy-btn" type="button" onclick="${COPY_ONCLICK}">Copy</button><pre><code class="language-${language}">${highlighted}</code></pre></figure>`;
     },
   },
 });
 
 /**
- * Convert markdown text to HTML with Highlight.js syntax highlighting.
+ * Convert markdown text to HTML with Prism.js syntax highlighting.
  * Unescapes HTML entities like `&amp;` \u2192 `&`.
  */
 export async function convertMarkdownToHTMLEscaped(