chat-agent-toolkit 1.2.49 → 1.2.51

package/dist/utils/markdown-to-html.d.ts

@@ -1,5 +1,5 @@
 /**
- * Convert markdown text to HTML with Highlight.js syntax highlighting.
+ * Convert markdown text to HTML with Prism.js syntax highlighting.
  * Unescapes HTML entities like `&` \u2192 `&`.
  */
 export declare function convertMarkdownToHTMLEscaped(markdown: string): Promise<string>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "chat-agent-toolkit",
-  "version": "1.2.49",
+  "version": "1.2.51",
   "description": "Multi-provider AI agent toolkit: generate language responses, search the web, extract content, and manage memory across 10+ LLM providers.",
   "author": "vtempest <grokthiscontact@gmail.com>",
   "license": "AGPL-3.0",
@@ -81,7 +81,7 @@
     "@mastra/memory": "^1.22.2",
     "ai": "^5.0.0",
     "drizzle-orm": "^0.45.1",
-    "highlight.js": "^11.11.1",
+    "prismjs": "^1.30.0",
     "html-entities": "^2.6.0",
     "manage-storage": "^0.0.13",
     "marked": "^17.0.4",

package/src/index.ts

@@ -20,6 +20,7 @@ export * from "./tools";
 // Both write-language and ./tools export AGENT_TOOLS; this package's own wins.
 export { AGENT_TOOLS } from "./tools";
 export * from "./utils";
+export * from "./mastra";
 export { configManager, ModelRegistry, getEnv, getModelProvidersUIConfigSection } from "./config";
 export type { Config, ConfigModelProvider, MCPServerConfig, UIConfigSections, Model, ModelWithProvider } from "./config";
 export { cropProvider, cropProviderAsBlob, cropProviderAsDataURL, getProviderImage, getProviderNames } from "./utils/provider-image-cropper";

package/src/mastra/agents.ts

@@ -0,0 +1,109 @@
+/**
+ * @fileoverview Mastra Agent Factory
+ *
+ * Creates typed Mastra agents with tool-calling, model selection,
+ * and optional memory integration.
+ */
+
+import { Agent } from "@mastra/core/agent";
+import { createTool } from "@mastra/core/tools";
+import { z } from "zod";
+
+export interface MastraAgentConfig {
+  id: string;
+  name: string;
+  instructions: string;
+  model: any;
+  tools?: Record<string, any>;
+  maxSteps?: number;
+  memory?: {
+    enabled: boolean;
+    threadId?: string;
+    userId?: string;
+  };
+}
+
+/**
+ * Create a Mastra agent with full configuration.
+ * Supports tool-calling, multi-step reasoning, and memory.
+ *
+ * @example
+ * ```ts
+ * import { createMastraAgent } from "chat-agent-toolkit/mastra";
+ * import { openai } from "@ai-sdk/openai";
+ *
+ * const agent = createMastraAgent({
+ *   id: "research-assistant",
+ *   name: "Research Assistant",
+ *   instructions: "Help users research topics thoroughly.",
+ *   model: openai("gpt-4o"),
+ *   tools: { webSearch, extractPage },
+ * });
+ *
+ * const result = await agent.generate("What is RAG?");
+ * ```
+ */
+export function createMastraAgent(config: MastraAgentConfig): Agent {
+  return new Agent({
+    id: config.id,
+    name: config.name,
+    instructions: config.instructions,
+    model: config.model,
+    tools: config.tools || {},
+  });
+}
+
+/**
+ * Create a Mastra agent with inline tool definitions using Zod schemas.
+ * Convenience wrapper for quick prototyping.
+ *
+ * @example
+ * ```ts
+ * const agent = createToolAgent({
+ *   id: "calculator",
+ *   name: "Calculator",
+ *   instructions: "Perform math operations.",
+ *   model: openai("gpt-4o"),
+ *   toolDefs: [{
+ *     id: "add",
+ *     description: "Add two numbers",
+ *     inputSchema: z.object({ a: z.number(), b: z.number() }),
+ *     outputSchema: z.object({ result: z.number() }),
+ *     execute: async ({ context }) => ({ result: context.a + context.b }),
+ *   }],
+ * });
+ * ```
+ */
+export function createToolAgent(config: {
+  id: string;
+  name: string;
+  instructions: string;
+  model: any;
+  toolDefs: Array<{
+    id: string;
+    description: string;
+    inputSchema: z.ZodType;
+    outputSchema: z.ZodType;
+    execute: (params: { context: any }) => Promise<any>;
+  }>;
+}): Agent {
+  const tools: Record<string, any> = {};
+
+  for (const def of config.toolDefs) {
+    tools[def.id] = createTool({
+      id: def.id,
+      description: def.description,
+      inputSchema: def.inputSchema,
+      outputSchema: def.outputSchema,
+      execute: def.execute,
+    });
+  }
+
+  return new Agent({
+    id: config.id,
+    name: config.name,
+    instructions: config.instructions,
+    model: config.model,
+    tools,
+  });
+}

package/src/mastra/evals.ts

@@ -0,0 +1,222 @@
+/**
+ * @fileoverview Mastra Evals
+ *
+ * Scorer-based evaluation functions for LLM outputs.
+ * Assess factuality, relevance, coherence, and safety.
+ */
+
+export interface EvalResult {
+  score: number;
+  reason: string;
+}
+
+export interface EvalSuiteResult {
+  overall: number;
+  results: Record<string, EvalResult>;
+}
+
+/**
+ * Evaluate factuality/groundedness of a response.
+ * Checks whether the output references sources or evidence.
+ *
+ * @example
+ * ```ts
+ * const result = await factualityEval(
+ *   "The earth orbits the sun (source: NASA).",
+ *   "What does the earth orbit?"
+ * );
+ * // { score: 0.9, reason: "Response includes source attribution." }
+ * ```
+ */
+export async function factualityEval(
+  output: string,
+  _query?: string,
+  sources?: string[]
+): Promise<EvalResult> {
+  const hasSourceRef = /\b(source|according to|reference|cited?|per)\b/i.test(output);
+  const hasUrl = /https?:\/\/\S+/.test(output);
+  const matchesSources = sources?.some((s) =>
+    output.toLowerCase().includes(s.toLowerCase().slice(0, 30))
+  );
+
+  let score = 0.3;
+  const reasons: string[] = [];
+
+  if (hasSourceRef) {
+    score += 0.25;
+    reasons.push("includes source attribution");
+  }
+  if (hasUrl) {
+    score += 0.15;
+    reasons.push("contains URL reference");
+  }
+  if (matchesSources) {
+    score += 0.3;
+    reasons.push("matches provided source material");
+  }
+  if (!hasSourceRef && !hasUrl && !matchesSources) {
+    reasons.push("no grounding evidence found");
+  }
+
+  return {
+    score: Math.min(score, 1.0),
+    reason: reasons.join("; ") + ".",
+  };
+}
+
+/**
+ * Evaluate relevance of a response to the query.
+ * Uses keyword overlap and length heuristics.
+ */
+export async function relevanceEval(
+  output: string,
+  query: string
+): Promise<EvalResult> {
+  const queryWords = new Set(
+    query.toLowerCase().split(/\s+/).filter((w) => w.length > 2)
+  );
+  const outputWords = output.toLowerCase().split(/\s+/);
+  const matchCount = outputWords.filter((w) => queryWords.has(w)).length;
+  const overlapRatio = queryWords.size > 0 ? matchCount / queryWords.size : 0;
+
+  const isTooShort = output.length < 20;
+  const isTooLong = output.length > query.length * 50;
+
+  let score = Math.min(overlapRatio * 1.5, 0.7);
+  const reasons: string[] = [];
+
+  if (overlapRatio > 0.5) {
+    score += 0.2;
+    reasons.push("high keyword overlap with query");
+  } else if (overlapRatio > 0.2) {
+    reasons.push("moderate keyword overlap");
+  } else {
+    reasons.push("low keyword overlap with query");
+  }
+
+  if (isTooShort) {
+    score -= 0.2;
+    reasons.push("response too short");
+  }
+  if (isTooLong) {
+    score -= 0.1;
+    reasons.push("response excessively long");
+  }
+
+  if (!isTooShort && !isTooLong && overlapRatio > 0.3) {
+    score += 0.1;
+  }
+
+  return {
+    score: Math.max(0, Math.min(score, 1.0)),
+    reason: reasons.join("; ") + ".",
+  };
+}
+
+/**
+ * Evaluate coherence and structure of a response.
+ * Checks sentence structure, paragraph flow, and logical markers.
+ */
+export async function coherenceEval(output: string): Promise<EvalResult> {
+  const sentences = output.split(/[.!?]+/).filter((s) => s.trim().length > 5);
+  const hasLogicalMarkers = /\b(therefore|because|however|furthermore|additionally|first|second|finally)\b/i.test(output);
+  const hasParagraphs = output.includes("\n\n") || output.includes("\n");
+  const avgSentenceLength =
+    sentences.length > 0
+      ? sentences.reduce((sum, s) => sum + s.trim().split(/\s+/).length, 0) / sentences.length
+      : 0;
+
+  let score = 0.4;
+  const reasons: string[] = [];
+
+  if (sentences.length >= 2) {
+    score += 0.15;
+    reasons.push("multiple sentences");
+  }
+  if (hasLogicalMarkers) {
+    score += 0.2;
+    reasons.push("uses logical connectors");
+  }
+  if (hasParagraphs && output.length > 200) {
+    score += 0.1;
+    reasons.push("structured with paragraphs");
+  }
+  if (avgSentenceLength > 5 && avgSentenceLength < 40) {
+    score += 0.15;
+    reasons.push("appropriate sentence length");
+  }
+  if (sentences.length < 2 && output.length > 100) {
+    score -= 0.2;
+    reasons.push("run-on text without sentence breaks");
+  }
+
+  return {
+    score: Math.max(0, Math.min(score, 1.0)),
+    reason: reasons.join("; ") + ".",
+  };
+}
+
+/**
+ * Evaluate output for toxic or harmful content.
+ * Basic keyword/pattern-based detection.
+ */
+export async function toxicityEval(output: string): Promise<EvalResult> {
+  const toxicPatterns = [
+    /\b(hate|kill|destroy|attack)\s+(all|every|them)\b/i,
+    /\b(stupid|idiot|moron|dumb)\s+(people|person|users?)\b/i,
+    /\b(should\s+die|deserve\s+to\s+die)\b/i,
+  ];
+
+  const matches = toxicPatterns.filter((p) => p.test(output));
+  const score = matches.length === 0 ? 1.0 : Math.max(0, 1.0 - matches.length * 0.4);
+
+  return {
+    score,
+    reason:
+      matches.length === 0
+        ? "No toxic patterns detected."
+        : `Detected ${matches.length} potentially harmful pattern(s).`,
+  };
+}
+
+/**
+ * Run a full evaluation suite against a response.
+ *
+ * @example
+ * ```ts
+ * const results = await runEvalSuite({
+ *   output: agentResponse,
+ *   query: "What is photosynthesis?",
+ *   sources: ["Biology textbook chapter 3"],
+ * });
+ *
+ * console.log(results.overall); // 0.78
+ * console.log(results.results.factuality.score); // 0.85
+ * ```
+ */
+export async function runEvalSuite(params: {
+  output: string;
+  query?: string;
+  sources?: string[];
+}): Promise<EvalSuiteResult> {
+  const { output, query = "", sources } = params;
+
+  const [factuality, relevance, coherence, toxicity] = await Promise.all([
+    factualityEval(output, query, sources),
+    relevanceEval(output, query),
+    coherenceEval(output),
+    toxicityEval(output),
+  ]);
+
+  const results: Record<string, EvalResult> = {
+    factuality,
+    relevance,
+    coherence,
+    toxicity,
+  };
+
+  const scores = Object.values(results).map((r) => r.score);
+  const overall = scores.reduce((sum, s) => sum + s, 0) / scores.length;
+
+  return { overall, results };
+}

package/src/mastra/index.ts

@@ -0,0 +1,45 @@
+/**
+ * @fileoverview Mastra Integration Module
+ *
+ * Full-featured Mastra AI framework integration providing agents, workflows,
+ * RAG, evals, telemetry, and model routing. Built on @mastra/core with
+ * Vercel AI SDK compatibility.
+ *
+ * @module mastra
+ */
+
+export { createMastraAgent, createToolAgent, type MastraAgentConfig } from "./agents";
+export {
+  createWorkflowStep,
+  createResearchWorkflow,
+  createRAGWorkflow,
+  type WorkflowStepConfig,
+} from "./workflows";
+export {
+  MastraRAG,
+  createRAGPipeline,
+  type RAGConfig,
+  type RAGDocument,
+  type RAGChunk,
+  type RAGRetrievalResult,
+} from "./rag";
+export {
+  factualityEval,
+  relevanceEval,
+  coherenceEval,
+  toxicityEval,
+  runEvalSuite,
+  type EvalResult,
+  type EvalSuiteResult,
+} from "./evals";
+export {
+  createMastraInstance,
+  type MastraInstanceConfig,
+  type TelemetryConfig,
+} from "./telemetry";
+export {
+  createModelRouter,
+  loadMastraModel,
+  type ModelRouterConfig,
+  type RoutingStrategy,
+} from "./model-routing";

package/src/mastra/model-routing.ts

@@ -0,0 +1,166 @@
+/**
+ * @fileoverview Mastra Model Routing
+ *
+ * Multi-provider model routing with strategy selection:
+ * cost-optimized, latency-optimized, capability-based, or round-robin.
+ */
+
+import { Agent } from "@mastra/core/agent";
+
+export type RoutingStrategy = "cost" | "latency" | "capability" | "round-robin";
+
+export interface ModelRouterConfig {
+  providers: Array<{
+    id: string;
+    model: any;
+    costPer1kTokens?: number;
+    avgLatencyMs?: number;
+    capabilities?: string[];
+    priority?: number;
+  }>;
+  strategy: RoutingStrategy;
+  fallbackProviderId?: string;
+}
+
+/**
+ * Model router that selects providers based on configurable strategies.
+ *
+ * @example
+ * ```ts
+ * import { createModelRouter } from "chat-agent-toolkit/mastra";
+ * import { openai } from "@ai-sdk/openai";
+ * import { anthropic } from "@ai-sdk/anthropic";
+ *
+ * const router = createModelRouter({
+ *   providers: [
+ *     { id: "openai", model: openai("gpt-4o"), costPer1kTokens: 0.005 },
+ *     { id: "anthropic", model: anthropic("claude-sonnet-4-20250514"), costPer1kTokens: 0.003 },
+ *     { id: "fast", model: openai("gpt-4o-mini"), costPer1kTokens: 0.0002, avgLatencyMs: 200 },
+ *   ],
+ *   strategy: "cost",
+ * });
+ *
+ * const model = router.select(); // cheapest provider
+ * const model2 = router.select({ capabilities: ["vision"] }); // capability match
+ * ```
+ */
+export function createModelRouter(config: ModelRouterConfig) {
+  let roundRobinIndex = 0;
+
+  return {
+    select(options?: {
+      capabilities?: string[];
+      maxCost?: number;
+      maxLatency?: number;
+    }): any {
+      let candidates = [...config.providers];
+
+      if (options?.capabilities) {
+        candidates = candidates.filter((p) =>
+          options.capabilities!.every(
+            (cap) => p.capabilities?.includes(cap)
+          )
+        );
+      }
+
+      if (options?.maxCost) {
+        candidates = candidates.filter(
+          (p) => (p.costPer1kTokens ?? Infinity) <= options.maxCost!
+        );
+      }
+
+      if (options?.maxLatency) {
+        candidates = candidates.filter(
+          (p) => (p.avgLatencyMs ?? Infinity) <= options.maxLatency!
+        );
+      }
+
+      if (candidates.length === 0) {
+        const fallback = config.providers.find(
+          (p) => p.id === config.fallbackProviderId
+        );
+        if (fallback) return fallback.model;
+        return config.providers[0]?.model;
+      }
+
+      switch (config.strategy) {
+        case "cost":
+          candidates.sort(
+            (a, b) => (a.costPer1kTokens ?? 0) - (b.costPer1kTokens ?? 0)
+          );
+          return candidates[0].model;
+
+        case "latency":
+          candidates.sort(
+            (a, b) => (a.avgLatencyMs ?? 0) - (b.avgLatencyMs ?? 0)
+          );
+          return candidates[0].model;
+
+        case "capability":
+          candidates.sort(
+            (a, b) => (b.capabilities?.length ?? 0) - (a.capabilities?.length ?? 0)
+          );
+          return candidates[0].model;
+
+        case "round-robin":
+          const selected = candidates[roundRobinIndex % candidates.length];
+          roundRobinIndex++;
+          return selected.model;
+
+        default:
+          return candidates[0].model;
+      }
+    },
+
+    createAgent(
+      agentConfig: { id: string; name: string; instructions: string; tools?: Record<string, any> },
+      routingOptions?: { capabilities?: string[]; maxCost?: number; maxLatency?: number }
+    ): Agent {
+      const model = this.select(routingOptions);
+      return new Agent({
+        id: agentConfig.id,
+        name: agentConfig.name,
+        instructions: agentConfig.instructions,
+        model,
+        tools: agentConfig.tools || {},
+      });
+    },
+  };
+}
+
+/**
+ * Load a Mastra-compatible model from a provider string.
+ * Convenience wrapper around Vercel AI SDK provider imports.
+ *
+ * @example
+ * ```ts
+ * const model = await loadMastraModel("openai", "gpt-4o");
+ * const model2 = await loadMastraModel("anthropic", "claude-sonnet-4-20250514");
+ * const model3 = await loadMastraModel("groq", "llama-3.3-70b-versatile");
+ * ```
+ */
+export async function loadMastraModel(
+  provider: "openai" | "anthropic" | "groq" | "google",
+  modelId: string
+): Promise<any> {
+  switch (provider) {
+    case "openai": {
+      const { openai } = await import("@ai-sdk/openai");
+      return openai(modelId);
+    }
+    case "anthropic": {
+      const { anthropic } = await import("@ai-sdk/anthropic");
+      return anthropic(modelId);
+    }
+    case "groq": {
+      const { createGroq } = await import("@ai-sdk/groq");
+      return createGroq({})(modelId);
+    }
+    case "google": {
+      const { google } = await import("@ai-sdk/google");
+      return google(modelId);
+    }
+    default:
+      throw new Error(`Unsupported provider: ${provider}`);
+  }
+}