@mcpmesh/sdk 1.4.1 → 2.0.0-beta.1

package/dist/llm-agent.js

@@ -4,20 +4,18 @@
  * This class handles:
  * - System prompt rendering (with Handlebars templates)
  * - Agentic loop with tool execution
- * - LLM provider calls (direct Vercel AI SDK or mesh delegation)
+ * - LLM provider calls via mesh delegation
  * - Response parsing with Zod validation
  * - Metadata tracking (tokens, latency, tool calls)
  *
  * Configuration Hierarchy (ENV > Config):
- * - MESH_LLM_PROVIDER: Override provider for direct mode (e.g., "claude", "openai", "gemini")
  * - MESH_LLM_MODEL: Override model (e.g., "gpt-4o", "gemini-2.0-flash")
  * - MESH_LLM_MAX_ITERATIONS: Override max agentic loop iterations
- * - MESH_LLM_FILTER_MODE: Override tool filter mode ("all", "include", "exclude")
  *
  * @example
  * ```typescript
  * const agent = new MeshLlmAgent({
- *   provider: "claude",
+ *   provider: { capability: "llm", tags: ["+claude"] },
  *   model: "anthropic/claude-sonnet-4-5",
  *   systemPrompt: "file://prompts/assistant.hbs",
  *   maxIterations: 10,
@@ -27,6 +25,7 @@
  * const result = await agent.run("Help me calculate 2+2", {
  *   templateContext: { user: "John" },
  *   tools: resolvedToolProxies,
+ *   meshProvider: { endpoint: "http://provider:9000", functionName: "process_chat" },
  * });
  * ```
  */
@@ -35,259 +34,11 @@ import { renderTemplate } from "./template.js";
 import { ResponseParser } from "./response-parser.js";
 import { MaxIterationsError, LLMAPIError, ToolExecutionError, } from "./errors.js";
 import { parseSSEResponse } from "./sse.js";
-import { loadProvider, extractVendorFromModel, extractModelName, } from "./llm-provider.js";
-import { ProviderHandlerRegistry } from "./provider-handlers/index.js";
 import { resolveMediaInputs } from "./media/index.js";
-import { getCurrentTraceContext, getCurrentPropagatedHeaders } from "./proxy.js";
+import { getCurrentTraceContext, getCurrentPropagatedHeaders, streamMcpTool, DEFAULT_CALL_OPTIONS, } from "./proxy.js";
 import { generateSpanId, publishTraceSpan, createTraceHeaders, injectTraceContext, } from "./tracing.js";
 import { fetchWithTimeout, isTimeoutError } from "./timeout-utils.js";
 import { getDispatcher } from "./http-pool.js";
-/**
- * Default LiteLLM provider using HTTP proxy.
- * Assumes LiteLLM proxy is running at LITELLM_URL or localhost:4000.
- */
-export class LiteLLMProvider {
-    baseUrl;
-    constructor(baseUrl) {
-        this.baseUrl = baseUrl || process.env.LITELLM_URL || "http://localhost:4000";
-    }
-    async complete(model, messages, tools, options) {
-        const body = {
-            model,
-            messages,
-        };
-        if (tools && tools.length > 0) {
-            body.tools = tools;
-            body.tool_choice = "auto";
-        }
-        if (options?.maxOutputTokens)
-            body.max_tokens = options.maxOutputTokens;
-        if (options?.temperature !== undefined)
-            body.temperature = options.temperature;
-        if (options?.topP !== undefined)
-            body.top_p = options.topP;
-        if (options?.stop)
-            body.stop = options.stop;
-        // Set up timeout (default 300s to match Python SDK's stream_timeout)
-        const timeoutMs = parseInt(process.env.LITELLM_TIMEOUT_MS || "300000", 10);
-        let response;
-        try {
-            response = await fetchWithTimeout(`${this.baseUrl}/v1/chat/completions`, {
-                method: "POST",
-                headers: {
-                    "Content-Type": "application/json",
-                },
-                body: JSON.stringify(body),
-                timeout: timeoutMs,
-                // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                dispatcher: getDispatcher(`${this.baseUrl}/v1/chat/completions`),
-            });
-        }
-        catch (error) {
-            if (isTimeoutError(error)) {
-                throw new LLMAPIError(408, `Request timed out after ${timeoutMs}ms`, "litellm");
-            }
-            throw new LLMAPIError(0, `Fetch failed: ${error instanceof Error ? error.message : String(error)}`, "litellm");
-        }
-        if (!response.ok) {
-            const error = await response.text();
-            throw new LLMAPIError(response.status, error, "litellm");
-        }
-        return (await response.json());
-    }
-}
-/**
- * Default model mappings for provider shorthand names.
- */
-const DEFAULT_MODELS = {
-    claude: "anthropic/claude-sonnet-4-5",
-    openai: "openai/gpt-4o",
-    anthropic: "anthropic/claude-sonnet-4-5",
-    gemini: "google/gemini-3-flash-preview",
-    google: "google/gemini-3-flash-preview",
-    gpt4: "openai/gpt-4o",
-    gpt35: "openai/gpt-3.5-turbo",
-};
-/**
- * Direct Vercel AI SDK provider.
- * Uses Vercel AI SDK (@ai-sdk/anthropic, @ai-sdk/openai, etc.) directly
- * without needing a proxy server.
- */
-export class VercelDirectProvider {
-    providerSpec;
-    cachedProvider = null;
-    providerLoadAttempted = false;
-    toolProxies = new Map();
-    maxSteps = 10;
-    constructor(providerSpec) {
-        this.providerSpec = providerSpec;
-    }
-    /**
-     * Set tool proxies for execute callbacks in the Vercel AI SDK agentic loop.
-     * When set, tools are created with execute callbacks and maxSteps is enabled,
-     * letting the SDK handle the tool execution loop internally.
-     */
-    setToolProxies(tools, maxIterations) {
-        this.toolProxies.clear();
-        for (const tool of tools) {
-            this.toolProxies.set(tool.name, tool);
-        }
-        this.maxSteps = maxIterations ?? 10;
-    }
-    /**
-     * Resolve the full model string from provider spec.
-     * E.g., "claude" -> "anthropic/claude-sonnet-4-5"
-     */
-    resolveModel(model) {
-        // If explicit model provided, use it
-        if (model && model !== "default") {
-            // If model already has vendor prefix, use as-is
-            if (model.includes("/")) {
-                return model;
-            }
-            // Otherwise, try to add vendor prefix from provider spec
-            const vendor = extractVendorFromModel(DEFAULT_MODELS[this.providerSpec.toLowerCase()] ?? this.providerSpec);
-            if (vendor) {
-                return `${vendor}/${model}`;
-            }
-            return model;
-        }
-        // Map shorthand provider to full model
-        const defaultModel = DEFAULT_MODELS[this.providerSpec.toLowerCase()];
-        if (defaultModel) {
-            return defaultModel;
-        }
-        // Assume provider spec is already a model identifier
-        return this.providerSpec;
-    }
-    async complete(model, messages, tools, options) {
-        const fullModel = this.resolveModel(model);
-        const vendor = extractVendorFromModel(fullModel);
-        const modelName = extractModelName(fullModel);
-        if (!vendor) {
-            throw new LLMAPIError(400, `Cannot determine vendor from model: ${fullModel}. Use format "vendor/model" (e.g., "anthropic/claude-sonnet-4-5")`, "vercel");
-        }
-        // Load provider if not cached
-        if (!this.providerLoadAttempted) {
-            this.providerLoadAttempted = true;
-            this.cachedProvider = await loadProvider(vendor);
-        }
-        if (!this.cachedProvider) {
-            throw new LLMAPIError(500, `Vercel AI SDK provider for '${vendor}' not available. Install: npm install @ai-sdk/${vendor}`, "vercel");
-        }
-        // Create the model instance
-        const aiModel = this.cachedProvider(modelName);
-        // Get vendor-specific handler for optimizations
-        const handler = ProviderHandlerRegistry.getHandler(vendor);
-        // Import generateText from ai package
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const aiModule = (await import("ai"));
-        const generateText = aiModule.generateText;
-        const jsonSchema = aiModule.jsonSchema;
-        const aiTool = aiModule.tool;
-        // Convert tools to Vercel AI SDK format
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        let vercelTools;
-        if (tools && tools.length > 0) {
-            vercelTools = {};
-            for (const tool of tools) {
-                const rawSchema = tool.function.parameters ?? {
-                    type: "object",
-                    properties: {},
-                };
-                // eslint-disable-next-line @typescript-eslint/no-unused-vars
-                const { $schema, ...schemaWithoutMeta } = rawSchema;
-                const cleanSchema = {
-                    type: "object",
-                    ...schemaWithoutMeta,
-                };
-                const proxy = this.toolProxies.get(tool.function.name);
-                vercelTools[tool.function.name] = aiTool({
-                    description: tool.function.description ?? "",
-                    inputSchema: jsonSchema(cleanSchema),
-                    ...(proxy ? {
-                        execute: async (args) => {
-                            const result = await proxy(args);
-                            return typeof result === "string" ? result : JSON.stringify(result);
-                        },
-                    } : {}),
-                });
-            }
-        }
-        // Apply vendor-specific request preparation
-        const preparedRequest = handler.prepareRequest(messages, null, // tools handled separately
-        options?.outputSchema ?? null, {
-            temperature: options?.temperature,
-            maxOutputTokens: options?.maxOutputTokens,
-            topP: options?.topP,
-        });
-        // Build request options
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const requestOptions = {
-            model: aiModel,
-            messages: preparedRequest.messages,
-        };
-        if (vercelTools && Object.keys(vercelTools).length > 0) {
-            requestOptions.tools = vercelTools;
-            // When tool proxies are set, the SDK handles the agentic loop via execute callbacks.
-            // maxSteps allows the SDK to call tools and feed results back to the LLM automatically.
-            if (this.toolProxies.size > 0) {
-                requestOptions.maxSteps = this.maxSteps;
-            }
-        }
-        if (options?.maxOutputTokens) {
-            requestOptions.maxTokens = options.maxOutputTokens;
-        }
-        if (options?.temperature !== undefined) {
-            requestOptions.temperature = options.temperature;
-        }
-        if (options?.topP !== undefined) {
-            requestOptions.topP = options.topP;
-        }
-        try {
-            const result = await generateText(requestOptions);
-            // Convert Vercel AI SDK response to LlmCompletionResponse format.
-            // When maxSteps is active, the SDK executed tools internally — don't
-            // expose intermediate tool_calls to the consumer's outer loop.
-            const sdkHandledLoop = requestOptions.maxSteps != null;
-            const response = {
-                id: `vercel-${Date.now()}`,
-                object: "chat.completion",
-                created: Math.floor(Date.now() / 1000),
-                model: fullModel,
-                choices: [
-                    {
-                        index: 0,
-                        message: {
-                            role: "assistant",
-                            content: result.text || null,
-                            tool_calls: sdkHandledLoop ? undefined : result.toolCalls?.map((tc) => ({
-                                id: tc.toolCallId,
-                                type: "function",
-                                function: {
-                                    name: tc.toolName,
-                                    arguments: JSON.stringify(tc.args ?? {}),
-                                },
-                            })),
-                        },
-                        finish_reason: result.finishReason ?? "stop",
-                    },
-                ],
-                usage: {
-                    prompt_tokens: result.usage?.promptTokens ?? 0,
-                    completion_tokens: result.usage?.completionTokens ?? 0,
-                    total_tokens: (result.usage?.promptTokens ?? 0) +
-                        (result.usage?.completionTokens ?? 0),
-                },
-            };
-            return response;
-        }
-        catch (error) {
-            const message = error instanceof Error ? error.message : String(error);
-            throw new LLMAPIError(500, `Vercel AI SDK error: ${message}`, "vercel");
-        }
-    }
-}
 /**
  * Mesh provider that delegates to an LLM provider discovered via mesh.
  */
@@ -486,6 +237,61 @@ export class MeshDelegatedProvider {
             }
         }
     }
+    /**
+     * Stream chunks from the mesh-delegated provider's streaming variant.
+     *
+     * Builds the same ``{request: <MeshLlmRequest>}`` body that ``complete()``
+     * produces, then calls the streaming MCP tool via ``streamMcpTool()`` from
+     * ``./proxy``. Each ``notifications/progress`` chunk is yielded as a string;
+     * the final ``result`` event ends the stream and is NOT yielded (matches the
+     * Python ``MeshLlmAgent.stream()`` contract).
+     *
+     * The provider's ``functionName`` is expected to already be the streaming
+     * variant — the registry resolver picks it based on the consumer's
+     * ``ai.mcpmesh.stream`` tag opt-in (see ``MeshLlmAgent.stream()``).
+     */
+    async *streamComplete(model, messages, tools, options) {
+        // Build MeshLlmRequest body — same shape as complete()
+        const modelParams = {};
+        if (model && model !== "default") {
+            modelParams.model = model;
+        }
+        if (options?.maxOutputTokens)
+            modelParams.max_tokens = options.maxOutputTokens;
+        if (options?.temperature !== undefined)
+            modelParams.temperature = options.temperature;
+        if (options?.topP !== undefined)
+            modelParams.top_p = options.topP;
+        if (options?.stop)
+            modelParams.stop = options.stop;
+        if (options?.outputSchema) {
+            modelParams.output_schema = options.outputSchema.schema;
+            modelParams.output_type_name = options.outputSchema.name;
+        }
+        if (this.parallelToolCalls) {
+            modelParams.parallel_tool_calls = true;
+        }
+        const request = { messages };
+        if (Object.keys(modelParams).length > 0) {
+            request.model_params = modelParams;
+        }
+        if (tools && tools.length > 0) {
+            request.tools = tools;
+        }
+        const args = { request };
+        // streamMcpTool() handles trace context injection / propagated headers /
+        // dispatcher pooling internally — same path as createProxy().stream().
+        // Match complete()'s env-backed timeout (MESH_PROVIDER_TIMEOUT_MS) so
+        // operators can tune both buffered and streaming provider calls with
+        // the same knob. Default 300s (matches Python SDK's stream_timeout).
+        const providerTimeoutMs = parseInt(process.env.MESH_PROVIDER_TIMEOUT_MS || "300000", 10);
+        const streamOptions = {
+            ...DEFAULT_CALL_OPTIONS,
+            timeout: providerTimeoutMs,
+            streamTimeout: providerTimeoutMs,
+        };
+        yield* streamMcpTool(this.endpoint, this.functionName, args, streamOptions, "mesh-llm-stream");
+    }
 }
 /**
  * MeshLlmAgent - The core agentic loop implementation.
@@ -633,11 +439,6 @@ export class MeshLlmAgent {
                 // If schema conversion fails, skip
             }
         }
-        // Set tool proxies on direct-mode providers so the Vercel AI SDK
-        // can execute tools internally via maxSteps (agentic loop in the SDK).
-        if (provider instanceof VercelDirectProvider && context.tools.length > 0) {
-            provider.setToolProxies(context.tools, this.config.maxIterations);
-        }
         // Agentic loop
         let iteration = 0;
         let finalContent = "";
@@ -731,6 +532,127 @@ export class MeshLlmAgent {
         // Parse and validate response
         return this.responseParser.parse(finalContent);
     }
+    /**
+     * Stream the final assistant text token-by-token from a mesh-delegated
+     * provider's streaming variant (Python's ``@mesh.llm_provider`` auto-
+     * generates a ``process_chat_stream`` MCP tool tagged
+     * ``ai.mcpmesh.stream``).
+     *
+     * **Tag opt-in (REQUIRED):** Unlike Python's ``@mesh.llm`` which auto-adds
+     * the ``ai.mcpmesh.stream`` tag based on the function's return-type
+     * (``Stream[str]`` vs ``str``), TypeScript users must EXPLICITLY include
+     * ``"ai.mcpmesh.stream"`` in their provider tag filter to get the
+     * streaming variant of the LLM provider:
+     *
+     * ```ts
+     * server.addTool(mesh.llm({
+     *   name: "chat_stream",
+     *   provider: { capability: "llm", tags: ["+claude", "ai.mcpmesh.stream"] },
+     *   // ...
+     *   execute: async ({ message }, { llm }) => {
+     *     for await (const chunk of llm.stream(message)) {
+     *       process.stdout.write(chunk);
+     *     }
+     *     return llm.meta?.outputTokens ? "ok" : "no-output";
+     *   },
+     * }));
+     * ```
+     *
+     * Without the ``ai.mcpmesh.stream`` tag the resolver returns the
+     * buffered ``process_chat`` tool, and ``stream()`` will yield zero chunks
+     * (the buffered tool emits no progress notifications).
+     *
+     * @param messageInput - User message string or multi-turn message array
+     * @param context - Runtime context with tools, mesh provider, and options
+     * @returns AsyncIterable yielding text chunks as the provider emits them
+     */
+    async *stream(messageInput, context) {
+        if (!context.meshProvider) {
+            throw new Error("MeshLlmAgent.stream() requires a mesh-delegated provider. " +
+                "Configure your agent with provider: { capability: 'llm', tags: ['ai.mcpmesh.stream'] } " +
+                "to use a streaming @mesh.llm_provider.");
+        }
+        // Build the same message list complete()/run() builds (system prompt,
+        // multipart media, multi-turn array unwinding) — without the agentic
+        // loop. The mesh-delegated streaming provider runs its own loop on the
+        // server side and emits text chunks via notifications/progress; the
+        // consumer just yields each one.
+        const messages = [];
+        const isMeshDelegated = true; // by definition: we required meshProvider above
+        const toolDefs = this.buildToolDefinitions(context.tools, isMeshDelegated);
+        // System prompt with template rendering + tool schema injection.
+        // Mirrors run(): mesh-delegated path skips the output-schema hint
+        // because the provider applies vendor-specific output formatting.
+        const systemPromptTemplate = this.getSystemPrompt();
+        if (systemPromptTemplate) {
+            let systemContent = await renderTemplate(systemPromptTemplate, context.templateContext ?? {});
+            if (toolDefs.length > 0) {
+                systemContent += this.buildToolSchemaSection(toolDefs);
+            }
+            messages.push({ role: "system", content: systemContent });
+        }
+        // Resolve media items to OpenAI-compatible image_url parts
+        const mediaItems = context.options?.media;
+        let mediaParts = null;
+        if (mediaItems && mediaItems.length > 0) {
+            mediaParts = await resolveMediaInputs(mediaItems);
+        }
+        if (typeof messageInput === "string") {
+            if (mediaParts && mediaParts.length > 0) {
+                messages.push({
+                    role: "user",
+                    content: [
+                        { type: "text", text: messageInput },
+                        ...mediaParts,
+                    ],
+                });
+            }
+            else {
+                messages.push({ role: "user", content: messageInput });
+            }
+        }
+        else {
+            for (let i = 0; i < messageInput.length; i++) {
+                const msg = messageInput[i];
+                const isLastUser = mediaParts &&
+                    mediaParts.length > 0 &&
+                    msg.role === "user" &&
+                    i === messageInput.length - 1;
+                if (isLastUser) {
+                    messages.push({
+                        role: "user",
+                        content: [
+                            { type: "text", text: msg.content },
+                            ...mediaParts,
+                        ],
+                    });
+                }
+                else {
+                    messages.push({ role: msg.role, content: msg.content });
+                }
+            }
+        }
+        // Effective options (runtime > env > config)
+        const maxTokens = context.options?.maxOutputTokens ?? this.config.maxOutputTokens;
+        const temperature = context.options?.temperature ?? this.config.temperature;
+        const model = context.meshProvider?.model ??
+            process.env.MESH_LLM_MODEL ??
+            this.config.model ??
+            this.getDefaultModel();
+        let outputSchema;
+        if (this.config.returnSchema) {
+            try {
+                const jsonSchema = zodToJsonSchema(this.config.returnSchema);
+                const schemaName = jsonSchema.title ?? "Response";
+                outputSchema = { schema: jsonSchema, name: schemaName };
+            }
+            catch {
+                // skip
+            }
+        }
+        const provider = new MeshDelegatedProvider(context.meshProvider.endpoint, context.meshProvider.functionName, this.config.parallelToolCalls ?? false);
+        yield* provider.streamComplete(model, messages, toolDefs.length > 0 ? toolDefs : undefined, { maxOutputTokens: maxTokens, temperature, topP: this.config.topP, stop: this.config.stop, outputSchema });
+    }
     /**
      * Create a callable LlmAgent interface.
      */
@@ -772,30 +694,44 @@ export class MeshLlmAgent {
         Object.defineProperty(callable, "setSystemPrompt", {
             value: (prompt) => agent.setSystemPrompt(prompt),
         });
+        // Attach stream method — async iterable for token-by-token output.
+        // Mirrors the callable's option-merging semantics so users get the same
+        // "context merge vs replace" behavior as the buffered call.
+        Object.defineProperty(callable, "stream", {
+            value: (message, options) => {
+                const contextMode = options?.contextMode ?? "merge";
+                let mergedTemplateContext;
+                if (contextMode === "replace" && options?.context) {
+                    mergedTemplateContext = options.context;
+                }
+                else if (options?.context) {
+                    mergedTemplateContext = { ...context.templateContext, ...options.context };
+                }
+                else {
+                    mergedTemplateContext = context.templateContext ?? {};
+                }
+                const mergedContext = {
+                    ...context,
+                    options: options ? { ...context.options, ...options } : context.options,
+                    templateContext: mergedTemplateContext,
+                };
+                return agent.stream(message, mergedContext);
+            },
+        });
         return callable;
     }
     /**
      * Resolve the LLM provider to use.
      *
-     * Configuration Hierarchy (ENV > Config):
-     * - MESH_LLM_PROVIDER: Override provider (only for direct mode, not mesh delegation)
+     * Mesh delegation only — a resolved meshProvider is required.
      */
     resolveProvider(context) {
-        // If mesh provider is resolved, use it (mesh delegation)
-        if (context.meshProvider) {
-            return new MeshDelegatedProvider(context.meshProvider.endpoint, context.meshProvider.functionName, this.config.parallelToolCalls ?? false);
+        if (!context.meshProvider) {
+            throw new Error("MeshLlmAgent requires a mesh-delegated provider. " +
+                "Configure your agent with provider: { capability: 'llm', tags: ['+claude'] } " +
+                "and ensure a matching @mesh.llm_provider is registered in the mesh.");
         }
-        // Use direct Vercel AI SDK provider
-        // Check env var override first (only for string provider, not mesh delegation object)
-        let providerSpec;
-        if (typeof this.config.provider === "string") {
-            providerSpec =
-                process.env.MESH_LLM_PROVIDER || this.config.provider || "claude";
-        }
-        else {
-            providerSpec = "claude"; // fallback default for non-mesh object config
-        }
-        return new VercelDirectProvider(providerSpec);
+        return new MeshDelegatedProvider(context.meshProvider.endpoint, context.meshProvider.functionName, this.config.parallelToolCalls ?? false);
     }
     /**
      * Get provider name for metadata.
@@ -804,29 +740,12 @@ export class MeshLlmAgent {
         if (context.meshProvider) {
             return `mesh:${context.meshProvider.endpoint}`;
         }
-        if (typeof this.config.provider === "string") {
-            // Return env var override if set, otherwise config value
-            return process.env.MESH_LLM_PROVIDER || this.config.provider;
-        }
         return `mesh:${this.config.provider.capability}`;
     }
     /**
-     * Get default model based on provider.
+     * Get default model — mesh delegation defers model selection to the provider.
      */
     getDefaultModel() {
-        const provider = this.config.provider;
-        if (typeof provider === "string") {
-            // Map common provider names to models
-            const defaultModels = {
-                claude: "anthropic/claude-sonnet-4-5",
-                openai: "gpt-4o",
-                anthropic: "anthropic/claude-sonnet-4-5",
-                gpt4: "gpt-4o",
-                gpt35: "gpt-3.5-turbo",
-            };
-            return defaultModels[provider.toLowerCase()] ?? provider;
-        }
-        // Mesh delegation - model will be determined by the provider
         return "default";
     }
     /**