@framers/agentos 0.9.26 → 0.9.28

package/dist/api/streamText.js

@@ -13,6 +13,7 @@ import { attachUsageAttributes, toTurnMetricUsage } from './observability.js';
 import { fireLlmUsageObserver } from './observers.js';
 import { hostPolicyToRouteParams, mergeRequiredCapabilities } from './runtime/hostPolicy.js';
 import { adaptTools } from './runtime/toolAdapter.js';
+import { runEmulatedToolLoop } from './runtime/tool-emulation/index.js';
 import { buildFallbackChain, createPlan, isRetryableError, resolveChainOfThought, } from './generateText.js';
 import { resolveDynamicToolCalls } from './runtime/dynamicToolCalling.js';
 import { StreamingReconstructor } from '../core/llm/streaming/StreamingReconstructor.js';
@@ -229,287 +230,354 @@ export function streamText(opts) {
                     rootSpan?.setAttribute('agentos.api.plan_steps', resolvedPlan.steps.length);
                 }
             }
-            for (let step = 0; step < maxSteps; step++) {
-                // --- onBeforeGeneration hook ---
-                let effectiveMessages = messages;
-                if (opts.onBeforeGeneration) {
-                    try {
-                        const hookCtx = {
-                            messages: [...messages],
-                            system: opts.system,
-                            tools: Array.from(toolMap.values()),
-                            model: resolved.modelId,
-                            provider: resolved.providerId,
-                            step,
-                            prompt: opts.prompt,
+            // --- Prompt-based tool-calling shim (toolMode) ---
+            // For models without native tool-use, buffer the tool roundtrips and emit
+            // the final answer as a single text part through the stream contract (spec
+            // decision #1 — replay from buffer, no extra model call). 'prompt' forces
+            // it up front; 'auto' tries native streaming first and falls back here on
+            // the provider's tool-unsupported error (see the catch around the loop).
+            const toolMode = opts.toolMode ?? 'auto';
+            const toolUnsupportedErr = (e) => e instanceof Error &&
+                /support tool use|does not support (tools|function)|no endpoints found that support/i.test(e.message);
+            async function* runShimStream() {
+                const loopResult = await runEmulatedToolLoop({
+                    tools: Array.from(toolMap.values()),
+                    messages: messages.map((m) => ({
+                        role: String(m.role),
+                        content: typeof m.content === 'string' ? m.content : JSON.stringify(m.content ?? ''),
+                    })),
+                    maxRoundtrips: opts.maxSteps ?? 5,
+                    callModel: async (msgs) => {
+                        // provider is guaranteed non-undefined by the `if (!provider) throw`
+                        // guard above; the closure just loses TS's flow-narrowing.
+                        const r = await provider.generateCompletion(resolved.modelId, msgs, {
+                            temperature: opts.temperature,
+                            maxTokens: opts.maxTokens,
+                        });
+                        const cc = r.choices?.[0]?.message?.content;
+                        return {
+                            text: typeof cc === 'string' ? cc : (cc?.text ?? ''),
+                            usage: { totalTokens: r.usage?.totalTokens ?? 0 },
                         };
-                        const modified = await opts.onBeforeGeneration(hookCtx);
-                        if (modified) {
-                            effectiveMessages = modified.messages;
-                        }
-                    }
-                    catch (hookErr) {
-                        console.warn('[agentos] onBeforeGeneration hook error:', hookErr);
-                    }
-                }
-                const stepSpan = startAgentOSSpan('agentos.api.stream_text.step', {
-                    attributes: {
-                        'llm.provider': resolved.providerId,
-                        'llm.model': resolved.modelId,
-                        'agentos.api.step': step + 1,
-                        'agentos.api.tool_count': tools.length,
                     },
                 });
-                const stream = provider.generateCompletionStream(resolved.modelId, effectiveMessages, {
-                    tools: toolSchemas,
-                    temperature: opts.temperature,
-                    maxTokens: opts.maxTokens,
-                });
-                const reconstructor = new StreamingReconstructor();
-                try {
-                    for await (const chunk of stream) {
-                        reconstructor.push(chunk);
-                        const textDelta = chunk.responseTextDelta ?? '';
-                        if (textDelta) {
-                            const part = { type: 'text', text: textDelta };
-                            parts.push(part);
-                            yield part;
-                        }
-                        if (chunk.error) {
-                            const error = new Error(chunk.error.message);
-                            const part = { type: 'error', error };
-                            parts.push(part);
-                            yield part;
-                            metricStatus = 'error';
-                            resolveText(finalText);
-                            resolveUsage(usage);
-                            resolveToolCalls(allToolCalls);
-                            return;
-                        }
-                        if (chunk.isFinal && chunk.usage) {
-                            usage.promptTokens += chunk.usage.promptTokens ?? 0;
-                            usage.completionTokens += chunk.usage.completionTokens ?? 0;
-                            usage.totalTokens += chunk.usage.totalTokens ?? 0;
-                            if (typeof chunk.usage.costUSD === 'number') {
-                                usage.costUSD = (usage.costUSD ?? 0) + chunk.usage.costUSD;
-                            }
-                            // Prompt-cache metrics from the provider layer. Provider
-                            // surfaces these as cacheReadInputTokens /
-                            // cacheCreationInputTokens on the final chunk's usage
-                            // (mirrors Anthropic's cache_read_input_tokens /
-                            // cache_creation_input_tokens); forward them onto the
-                            // aggregated TokenUsage so downstream cost trackers see
-                            // cache hits.
-                            const cacheRead = chunk.usage.cacheReadInputTokens;
-                            const cacheCreate = chunk.usage.cacheCreationInputTokens;
-                            if (typeof cacheRead === 'number') {
-                                usage.cacheReadTokens = (usage.cacheReadTokens ?? 0) + cacheRead;
-                            }
-                            if (typeof cacheCreate === 'number') {
-                                usage.cacheCreationTokens = (usage.cacheCreationTokens ?? 0) + cacheCreate;
+                usage.totalTokens = (usage.totalTokens ?? 0) + loopResult.totalTokens;
+                finalText = loopResult.text;
+                const shimToolCalls = loopResult.toolCalls.map((c) => ({
+                    name: c.name,
+                    args: c.args,
+                    ...(c.error ? { error: c.error } : {}),
+                }));
+                if (loopResult.text) {
+                    const part = { type: 'text', text: loopResult.text };
+                    parts.push(part);
+                    yield part;
+                }
+                resolveText(finalText);
+                resolveUsage(usage);
+                resolveToolCalls(shimToolCalls);
+            }
+            if (tools.length > 0 && toolMode === 'prompt') {
+                yield* runShimStream();
+                return;
+            }
+            let streamedAnyText = false;
+            try {
+                for (let step = 0; step < maxSteps; step++) {
+                    // --- onBeforeGeneration hook ---
+                    let effectiveMessages = messages;
+                    if (opts.onBeforeGeneration) {
+                        try {
+                            const hookCtx = {
+                                messages: [...messages],
+                                system: opts.system,
+                                tools: Array.from(toolMap.values()),
+                                model: resolved.modelId,
+                                provider: resolved.providerId,
+                                step,
+                                prompt: opts.prompt,
+                            };
+                            const modified = await opts.onBeforeGeneration(hookCtx);
+                            if (modified) {
+                                effectiveMessages = modified.messages;
                             }
-                            attachUsageAttributes(stepSpan, {
-                                promptTokens: chunk.usage.promptTokens,
-                                completionTokens: chunk.usage.completionTokens,
-                                totalTokens: chunk.usage.totalTokens,
-                                costUSD: chunk.usage.costUSD,
-                            });
+                        }
+                        catch (hookErr) {
+                            console.warn('[agentos] onBeforeGeneration hook error:', hookErr);
                         }
                     }
-                }
-                finally {
-                    stepSpan?.end();
-                }
-                const stepText = reconstructor.getFullText();
-                const finalChunk = reconstructor.getFinalChunk();
-                let streamedToolCalls = resolveDynamicToolCalls(finalChunk?.choices?.[0]?.message?.tool_calls ??
-                    reconstructor
-                        .getToolCalls()
-                        .filter((toolCall) => toolCall.id && toolCall.name)
-                        .map((toolCall) => ({
-                        id: toolCall.id,
-                        type: 'function',
-                        function: {
-                            name: toolCall.name,
-                            arguments: toolCall.rawArguments || JSON.stringify(toolCall.arguments ?? {}),
+                    const stepSpan = startAgentOSSpan('agentos.api.stream_text.step', {
+                        attributes: {
+                            'llm.provider': resolved.providerId,
+                            'llm.model': resolved.modelId,
+                            'agentos.api.step': step + 1,
+                            'agentos.api.tool_count': tools.length,
                         },
-                    })), {
-                    text: stepText,
-                    step,
-                    toolsAvailable: tools.length > 0,
-                });
-                // --- onAfterGeneration hook ---
-                let effectiveStepText = stepText;
-                if (opts.onAfterGeneration) {
+                    });
+                    const stream = provider.generateCompletionStream(resolved.modelId, effectiveMessages, {
+                        tools: toolSchemas,
+                        temperature: opts.temperature,
+                        maxTokens: opts.maxTokens,
+                    });
+                    const reconstructor = new StreamingReconstructor();
                     try {
-                        const stepUsage = {
-                            promptTokens: usage.promptTokens,
-                            completionTokens: usage.completionTokens,
-                            totalTokens: usage.totalTokens,
-                            costUSD: usage.costUSD,
-                            cacheReadTokens: usage.cacheReadTokens,
-                            cacheCreationTokens: usage.cacheCreationTokens,
-                        };
-                        const toolCallRecords = (streamedToolCalls ?? []).map((tc) => ({
-                            name: tc.function?.name ?? '',
-                            args: tc.function?.arguments ?? '{}',
-                        }));
-                        const hookResult = {
-                            text: stepText,
-                            toolCalls: toolCallRecords,
-                            usage: stepUsage,
-                            step,
-                        };
-                        const modified = await opts.onAfterGeneration(hookResult);
-                        if (modified) {
-                            effectiveStepText = modified.text;
-                            if (modified.toolCalls.length === 0 && streamedToolCalls && streamedToolCalls.length > 0) {
-                                streamedToolCalls = [];
+                        for await (const chunk of stream) {
+                            reconstructor.push(chunk);
+                            const textDelta = chunk.responseTextDelta ?? '';
+                            if (textDelta) {
+                                const part = { type: 'text', text: textDelta };
+                                parts.push(part);
+                                yield part;
+                                streamedAnyText = true;
+                            }
+                            if (chunk.error) {
+                                const error = new Error(chunk.error.message);
+                                const part = { type: 'error', error };
+                                parts.push(part);
+                                yield part;
+                                metricStatus = 'error';
+                                resolveText(finalText);
+                                resolveUsage(usage);
+                                resolveToolCalls(allToolCalls);
+                                return;
+                            }
+                            if (chunk.isFinal && chunk.usage) {
+                                usage.promptTokens += chunk.usage.promptTokens ?? 0;
+                                usage.completionTokens += chunk.usage.completionTokens ?? 0;
+                                usage.totalTokens += chunk.usage.totalTokens ?? 0;
+                                if (typeof chunk.usage.costUSD === 'number') {
+                                    usage.costUSD = (usage.costUSD ?? 0) + chunk.usage.costUSD;
+                                }
+                                // Prompt-cache metrics from the provider layer. Provider
+                                // surfaces these as cacheReadInputTokens /
+                                // cacheCreationInputTokens on the final chunk's usage
+                                // (mirrors Anthropic's cache_read_input_tokens /
+                                // cache_creation_input_tokens); forward them onto the
+                                // aggregated TokenUsage so downstream cost trackers see
+                                // cache hits.
+                                const cacheRead = chunk.usage.cacheReadInputTokens;
+                                const cacheCreate = chunk.usage.cacheCreationInputTokens;
+                                if (typeof cacheRead === 'number') {
+                                    usage.cacheReadTokens = (usage.cacheReadTokens ?? 0) + cacheRead;
+                                }
+                                if (typeof cacheCreate === 'number') {
+                                    usage.cacheCreationTokens = (usage.cacheCreationTokens ?? 0) + cacheCreate;
+                                }
+                                attachUsageAttributes(stepSpan, {
+                                    promptTokens: chunk.usage.promptTokens,
+                                    completionTokens: chunk.usage.completionTokens,
+                                    totalTokens: chunk.usage.totalTokens,
+                                    costUSD: chunk.usage.costUSD,
+                                });
                             }
                         }
                     }
-                    catch (hookErr) {
-                        console.warn('[agentos] onAfterGeneration hook error:', hookErr);
+                    finally {
+                        stepSpan?.end();
                     }
-                }
-                // Always track the latest step's text so finalText is available even
-                // when maxSteps is exhausted with outstanding tool calls.
-                if (effectiveStepText) {
-                    finalText = effectiveStepText;
-                }
-                if (!streamedToolCalls || streamedToolCalls.length === 0) {
-                    rootSpan?.setAttribute('agentos.api.finish_reason', 'stop');
-                    rootSpan?.setAttribute('agentos.api.tool_calls', allToolCalls.length);
-                    attachUsageAttributes(rootSpan, usage);
-                    resolveText(finalText);
-                    resolveUsage(usage);
-                    resolveToolCalls(allToolCalls);
-                    return;
-                }
-                messages.push({
-                    role: 'assistant',
-                    content: effectiveStepText || null,
-                    tool_calls: streamedToolCalls,
-                });
-                for (const toolCall of streamedToolCalls) {
-                    const fnName = toolCall.function?.name ?? '';
-                    const rawArgs = toolCall.function?.arguments ?? '{}';
-                    const toolCallId = toolCall.id ?? '';
-                    const toolCallRecord = {
-                        name: fnName,
-                        args: rawArgs,
-                    };
-                    let parsedArgs;
-                    try {
-                        parsedArgs = typeof rawArgs === 'string' ? JSON.parse(rawArgs) : rawArgs;
-                        toolCallRecord.args = parsedArgs;
-                    }
-                    catch {
-                        toolCallRecord.error = `Tool "${fnName}" arguments were not valid JSON.`;
-                        const resultPart = {
-                            type: 'tool-result',
-                            toolName: fnName,
-                            result: { error: toolCallRecord.error },
-                        };
-                        parts.push(resultPart);
-                        yield resultPart;
-                        messages.push({
-                            role: 'tool',
-                            tool_call_id: toolCallId,
-                            content: JSON.stringify({ error: toolCallRecord.error }),
-                        });
-                        allToolCalls.push(toolCallRecord);
-                        continue;
-                    }
-                    const requestPart = { type: 'tool-call', toolName: fnName, args: parsedArgs };
-                    parts.push(requestPart);
-                    yield requestPart;
-                    const tool = toolMap.get(fnName);
-                    if (!tool) {
-                        toolCallRecord.error = `Tool "${fnName}" not found.`;
-                        const resultPart = {
-                            type: 'tool-result',
-                            toolName: fnName,
-                            result: { error: toolCallRecord.error },
-                        };
-                        parts.push(resultPart);
-                        yield resultPart;
-                        messages.push({
-                            role: 'tool',
-                            tool_call_id: toolCallId,
-                            content: JSON.stringify({ error: toolCallRecord.error }),
-                        });
-                        allToolCalls.push(toolCallRecord);
-                        continue;
-                    }
-                    // --- onBeforeToolExecution hook ---
-                    if (opts.onBeforeToolExecution) {
+                    const stepText = reconstructor.getFullText();
+                    const finalChunk = reconstructor.getFinalChunk();
+                    let streamedToolCalls = resolveDynamicToolCalls(finalChunk?.choices?.[0]?.message?.tool_calls ??
+                        reconstructor
+                            .getToolCalls()
+                            .filter((toolCall) => toolCall.id && toolCall.name)
+                            .map((toolCall) => ({
+                            id: toolCall.id,
+                            type: 'function',
+                            function: {
+                                name: toolCall.name,
+                                arguments: toolCall.rawArguments || JSON.stringify(toolCall.arguments ?? {}),
+                            },
+                        })), {
+                        text: stepText,
+                        step,
+                        toolsAvailable: tools.length > 0,
+                    });
+                    // --- onAfterGeneration hook ---
+                    let effectiveStepText = stepText;
+                    if (opts.onAfterGeneration) {
                         try {
-                            const hookInfo = {
-                                name: fnName,
-                                args: parsedArgs,
-                                id: toolCallId || '',
+                            const stepUsage = {
+                                promptTokens: usage.promptTokens,
+                                completionTokens: usage.completionTokens,
+                                totalTokens: usage.totalTokens,
+                                costUSD: usage.costUSD,
+                                cacheReadTokens: usage.cacheReadTokens,
+                                cacheCreationTokens: usage.cacheCreationTokens,
+                            };
+                            const toolCallRecords = (streamedToolCalls ?? []).map((tc) => ({
+                                name: tc.function?.name ?? '',
+                                args: tc.function?.arguments ?? '{}',
+                            }));
+                            const hookResult = {
+                                text: stepText,
+                                toolCalls: toolCallRecords,
+                                usage: stepUsage,
                                 step,
                             };
-                            const hookResult = await opts.onBeforeToolExecution(hookInfo);
-                            if (hookResult === null) {
-                                toolCallRecord.error = 'Skipped by onBeforeToolExecution hook';
-                                const resultPart = {
-                                    type: 'tool-result',
-                                    toolName: fnName,
-                                    result: { skipped: true },
-                                };
-                                parts.push(resultPart);
-                                yield resultPart;
-                                messages.push({
-                                    role: 'tool',
-                                    tool_call_id: toolCallId,
-                                    content: JSON.stringify({ skipped: true }),
-                                });
-                                allToolCalls.push(toolCallRecord);
-                                continue;
+                            const modified = await opts.onAfterGeneration(hookResult);
+                            if (modified) {
+                                effectiveStepText = modified.text;
+                                if (modified.toolCalls.length === 0 && streamedToolCalls && streamedToolCalls.length > 0) {
+                                    streamedToolCalls = [];
+                                }
                             }
-                            parsedArgs = hookResult.args;
                         }
                         catch (hookErr) {
-                            console.warn('[agentos] onBeforeToolExecution hook error:', hookErr);
+                            console.warn('[agentos] onAfterGeneration hook error:', hookErr);
                         }
                     }
-                    try {
-                        const result = await tool.execute(parsedArgs, buildHelperToolExecutionContext('streamText', helperToolRunId, step, toolCallId || undefined));
-                        toolCallRecord.result = result.output;
-                        toolCallRecord.error = result.success ? undefined : result.error;
-                        const resultPart = {
-                            type: 'tool-result',
-                            toolName: fnName,
-                            result: result.output ?? { error: result.error },
-                        };
-                        parts.push(resultPart);
-                        yield resultPart;
-                        messages.push({
-                            role: 'tool',
-                            tool_call_id: toolCallId,
-                            content: JSON.stringify(result.output ?? { error: result.error ?? 'Tool execution failed.' }),
-                        });
+                    // Always track the latest step's text so finalText is available even
+                    // when maxSteps is exhausted with outstanding tool calls.
+                    if (effectiveStepText) {
+                        finalText = effectiveStepText;
+                    }
+                    if (!streamedToolCalls || streamedToolCalls.length === 0) {
+                        rootSpan?.setAttribute('agentos.api.finish_reason', 'stop');
+                        rootSpan?.setAttribute('agentos.api.tool_calls', allToolCalls.length);
+                        attachUsageAttributes(rootSpan, usage);
+                        resolveText(finalText);
+                        resolveUsage(usage);
+                        resolveToolCalls(allToolCalls);
+                        return;
                     }
-                    catch (err) {
-                        toolCallRecord.error = err?.message ?? String(err);
-                        const resultPart = {
-                            type: 'tool-result',
-                            toolName: fnName,
-                            result: { error: toolCallRecord.error },
+                    messages.push({
+                        role: 'assistant',
+                        content: effectiveStepText || null,
+                        tool_calls: streamedToolCalls,
+                    });
+                    for (const toolCall of streamedToolCalls) {
+                        const fnName = toolCall.function?.name ?? '';
+                        const rawArgs = toolCall.function?.arguments ?? '{}';
+                        const toolCallId = toolCall.id ?? '';
+                        const toolCallRecord = {
+                            name: fnName,
+                            args: rawArgs,
                         };
-                        parts.push(resultPart);
-                        yield resultPart;
-                        messages.push({
-                            role: 'tool',
-                            tool_call_id: toolCallId,
-                            content: JSON.stringify({ error: toolCallRecord.error }),
-                        });
+                        let parsedArgs;
+                        try {
+                            parsedArgs = typeof rawArgs === 'string' ? JSON.parse(rawArgs) : rawArgs;
+                            toolCallRecord.args = parsedArgs;
+                        }
+                        catch {
+                            toolCallRecord.error = `Tool "${fnName}" arguments were not valid JSON.`;
+                            const resultPart = {
+                                type: 'tool-result',
+                                toolName: fnName,
+                                result: { error: toolCallRecord.error },
+                            };
+                            parts.push(resultPart);
+                            yield resultPart;
+                            messages.push({
+                                role: 'tool',
+                                tool_call_id: toolCallId,
+                                content: JSON.stringify({ error: toolCallRecord.error }),
+                            });
+                            allToolCalls.push(toolCallRecord);
+                            continue;
+                        }
+                        const requestPart = { type: 'tool-call', toolName: fnName, args: parsedArgs };
+                        parts.push(requestPart);
+                        yield requestPart;
+                        const tool = toolMap.get(fnName);
+                        if (!tool) {
+                            toolCallRecord.error = `Tool "${fnName}" not found.`;
+                            const resultPart = {
+                                type: 'tool-result',
+                                toolName: fnName,
+                                result: { error: toolCallRecord.error },
+                            };
+                            parts.push(resultPart);
+                            yield resultPart;
+                            messages.push({
+                                role: 'tool',
+                                tool_call_id: toolCallId,
+                                content: JSON.stringify({ error: toolCallRecord.error }),
+                            });
+                            allToolCalls.push(toolCallRecord);
+                            continue;
+                        }
+                        // --- onBeforeToolExecution hook ---
+                        if (opts.onBeforeToolExecution) {
+                            try {
+                                const hookInfo = {
+                                    name: fnName,
+                                    args: parsedArgs,
+                                    id: toolCallId || '',
+                                    step,
+                                };
+                                const hookResult = await opts.onBeforeToolExecution(hookInfo);
+                                if (hookResult === null) {
+                                    toolCallRecord.error = 'Skipped by onBeforeToolExecution hook';
+                                    const resultPart = {
+                                        type: 'tool-result',
+                                        toolName: fnName,
+                                        result: { skipped: true },
+                                    };
+                                    parts.push(resultPart);
+                                    yield resultPart;
+                                    messages.push({
+                                        role: 'tool',
+                                        tool_call_id: toolCallId,
+                                        content: JSON.stringify({ skipped: true }),
+                                    });
+                                    allToolCalls.push(toolCallRecord);
+                                    continue;
+                                }
+                                parsedArgs = hookResult.args;
+                            }
+                            catch (hookErr) {
+                                console.warn('[agentos] onBeforeToolExecution hook error:', hookErr);
+                            }
+                        }
+                        try {
+                            const result = await tool.execute(parsedArgs, buildHelperToolExecutionContext('streamText', helperToolRunId, step, toolCallId || undefined));
+                            toolCallRecord.result = result.output;
+                            toolCallRecord.error = result.success ? undefined : result.error;
+                            const resultPart = {
+                                type: 'tool-result',
+                                toolName: fnName,
+                                result: result.output ?? { error: result.error },
+                            };
+                            parts.push(resultPart);
+                            yield resultPart;
+                            messages.push({
+                                role: 'tool',
+                                tool_call_id: toolCallId,
+                                content: JSON.stringify(result.output ?? { error: result.error ?? 'Tool execution failed.' }),
+                            });
+                        }
+                        catch (err) {
+                            toolCallRecord.error = err?.message ?? String(err);
+                            const resultPart = {
+                                type: 'tool-result',
+                                toolName: fnName,
+                                result: { error: toolCallRecord.error },
+                            };
+                            parts.push(resultPart);
+                            yield resultPart;
+                            messages.push({
+                                role: 'tool',
+                                tool_call_id: toolCallId,
+                                content: JSON.stringify({ error: toolCallRecord.error }),
+                            });
+                        }
+                        allToolCalls.push(toolCallRecord);
                     }
-                    allToolCalls.push(toolCallRecord);
                 }
             }
+            catch (loopErr) {
+                // 'auto' reactive fallback: if the provider rejected native tool-use
+                // before any text streamed, re-run the turn through the prompt shim.
+                if (!streamedAnyText &&
+                    tools.length > 0 &&
+                    toolMode === 'auto' &&
+                    toolUnsupportedErr(loopErr)) {
+                    yield* runShimStream();
+                    return;
+                }
+                throw loopErr;
+            }
             resolveText(finalText);
             resolveUsage(usage);
             resolveToolCalls(allToolCalls);