@alpic80/rivet-core 1.19.1-aidon.1

package/dist/esm/plugins/anthropic/nodes/ChatAnthropicNode.js

@@ -0,0 +1,652 @@
+import { uint8ArrayToBase64, } from '../../../index.js';
+import { anthropicModelOptions, anthropicModels, streamChatCompletions, AnthropicError, streamMessageApi, callMessageApi, } from '../anthropic.js';
+import { nanoid } from 'nanoid/non-secure';
+import { dedent } from 'ts-dedent';
+import retry from 'p-retry';
+import { match } from 'ts-pattern';
+import { coerceType, coerceTypeOptional } from '../../../utils/coerceType.js';
+import { addWarning } from '../../../utils/outputs.js';
+import { getError } from '../../../utils/errors.js';
+import { pluginNodeDefinition } from '../../../model/NodeDefinition.js';
+import { getScalarTypeOf, isArrayDataValue } from '../../../model/DataValue.js';
+import { assertNever } from '../../../utils/assertNever.js';
+import { isNotNull } from '../../../utils/genericUtilFunctions.js';
+// Temporary
+const cache = new Map();
+export const ChatAnthropicNodeImpl = {
+    create() {
+        const chartNode = {
+            type: 'chatAnthropic',
+            title: 'Chat (Anthropic)',
+            id: nanoid(),
+            visualData: {
+                x: 0,
+                y: 0,
+                width: 275,
+            },
+            data: {
+                model: 'claude-3-5-sonnet-20240620',
+                useModelInput: false,
+                temperature: 0.5,
+                useTemperatureInput: false,
+                top_p: 1,
+                useTopPInput: false,
+                top_k: undefined,
+                useTopKInput: false,
+                useTopP: false,
+                useUseTopPInput: false,
+                maxTokens: 1024,
+                useMaxTokensInput: false,
+                useStop: false,
+                stop: '',
+                useStopInput: false,
+                cache: false,
+                useAsGraphPartialOutput: true,
+                enableToolUse: false,
+            },
+        };
+        return chartNode;
+    },
+    getInputDefinitions(data) {
+        const inputs = [];
+        if (data.model.startsWith('claude-3')) {
+            inputs.push({
+                dataType: 'string',
+                id: 'system',
+                title: 'System Prompt',
+            });
+        }
+        if (data.useModelInput) {
+            inputs.push({
+                id: 'model',
+                title: 'Model',
+                dataType: 'string',
+                required: false,
+            });
+        }
+        if (data.useTemperatureInput) {
+            inputs.push({
+                dataType: 'number',
+                id: 'temperature',
+                title: 'Temperature',
+            });
+        }
+        if (data.useTopPInput) {
+            inputs.push({
+                dataType: 'number',
+                id: 'top_p',
+                title: 'Top P',
+            });
+        }
+        if (data.useUseTopPInput) {
+            inputs.push({
+                dataType: 'boolean',
+                id: 'useTopP',
+                title: 'Use Top P',
+            });
+        }
+        if (data.useMaxTokensInput) {
+            inputs.push({
+                dataType: 'number',
+                id: 'maxTokens',
+                title: 'Max Tokens',
+            });
+        }
+        if (data.useStopInput) {
+            inputs.push({
+                dataType: 'string',
+                id: 'stop',
+                title: 'Stop',
+            });
+        }
+        inputs.push({
+            dataType: ['chat-message', 'chat-message[]'],
+            id: 'prompt',
+            title: 'Prompt',
+        });
+        if (data.enableToolUse) {
+            inputs.push({
+                dataType: ['gpt-function', 'gpt-function[]'],
+                id: 'tools',
+                title: 'Tools',
+                description: 'Tools to use in the model. To connect multiple tools, use an Array node.',
+                coerced: false,
+            });
+        }
+        return inputs;
+    },
+    getOutputDefinitions(data) {
+        const outputs = [];
+        outputs.push({
+            dataType: 'string',
+            id: 'response',
+            title: 'Response',
+        });
+        if (data.enableToolUse) {
+            outputs.push({
+                dataType: 'object[]',
+                id: 'function-calls',
+                title: 'Function Calls',
+                description: 'The function calls that were made, if any.',
+            });
+        }
+        outputs.push({
+            dataType: 'chat-message[]',
+            id: 'all-messages',
+            title: 'All Messages',
+            description: 'All messages, with the response appended.',
+        });
+        return outputs;
+    },
+    getBody(data) {
+        const modelName = anthropicModels[data.model]?.displayName ?? 'Unknown Model';
+        return dedent `
+      ${modelName}
+      ${data.useTopP
+            ? `Top P: ${data.useTopPInput ? '(Using Input)' : data.top_p}`
+            : `Temperature: ${data.useTemperatureInput ? '(Using Input)' : data.temperature}`}
+      Max Tokens: ${data.maxTokens}
+      ${data.useStop ? `Stop: ${data.useStopInput ? '(Using Input)' : data.stop}` : ''}
+    `;
+    },
+    getEditors() {
+        return [
+            {
+                type: 'dropdown',
+                label: 'Model',
+                dataKey: 'model',
+                useInputToggleDataKey: 'useModelInput',
+                options: anthropicModelOptions,
+            },
+            {
+                type: 'number',
+                label: 'Temperature',
+                dataKey: 'temperature',
+                useInputToggleDataKey: 'useTemperatureInput',
+                min: 0,
+                max: 2,
+                step: 0.1,
+            },
+            {
+                type: 'number',
+                label: 'Top P',
+                dataKey: 'top_p',
+                useInputToggleDataKey: 'useTopPInput',
+                min: 0,
+                max: 1,
+                step: 0.1,
+            },
+            {
+                type: 'toggle',
+                label: 'Use Top P',
+                dataKey: 'useTopP',
+                useInputToggleDataKey: 'useUseTopPInput',
+            },
+            {
+                type: 'number',
+                label: 'Max Tokens',
+                dataKey: 'maxTokens',
+                useInputToggleDataKey: 'useMaxTokensInput',
+                min: 0,
+                max: Number.MAX_SAFE_INTEGER,
+                step: 1,
+            },
+            {
+                type: 'string',
+                label: 'Stop',
+                dataKey: 'stop',
+                useInputToggleDataKey: 'useStopInput',
+            },
+            {
+                type: 'toggle',
+                label: 'Cache (same inputs, same outputs)',
+                dataKey: 'cache',
+            },
+            {
+                type: 'toggle',
+                label: 'Use for subgraph partial output',
+                dataKey: 'useAsGraphPartialOutput',
+            },
+            {
+                type: 'toggle',
+                label: 'Enable Tool Use (disables streaming)',
+                dataKey: 'enableToolUse',
+            },
+        ];
+    },
+    getUIData() {
+        return {
+            infoBoxBody: dedent `
+        Makes a call to an Anthropic chat model. The settings contains many options for tweaking the model's behavior.
+      `,
+            infoBoxTitle: 'Chat (Anthropic) Node',
+            contextMenuTitle: 'Chat (Anthropic)',
+            group: ['AI'],
+        };
+    },
+    async process(data, inputs, context) {
+        if (context.executor === 'browser') {
+            throw new Error('This node requires using the Node executor');
+        }
+        const output = {};
+        const rawModel = data.useModelInput
+            ? coerceTypeOptional(inputs['model'], 'string') ?? data.model
+            : data.model;
+        const model = rawModel;
+        const temperature = data.useTemperatureInput
+            ? coerceTypeOptional(inputs['temperature'], 'number') ?? data.temperature
+            : data.temperature;
+        const topP = data.useTopPInput ? coerceTypeOptional(inputs['top_p'], 'number') ?? data.top_p : data.top_p;
+        const useTopP = data.useUseTopPInput
+            ? coerceTypeOptional(inputs['useTopP'], 'boolean') ?? data.useTopP
+            : data.useTopP;
+        const stop = data.useStopInput
+            ? data.useStop
+                ? coerceTypeOptional(inputs['stop'], 'string') ?? data.stop
+                : undefined
+            : data.stop;
+        const tools = data.enableToolUse
+            ? coerceTypeOptional(inputs['tools'], 'gpt-function[]') ?? []
+            : undefined;
+        const rivetChatMessages = getChatMessages(inputs);
+        const messages = await chatMessagesToClaude3ChatMessages(rivetChatMessages);
+        let prompt = messages.reduce((acc, message) => {
+            const content = typeof message.content === 'string'
+                ? message.content
+                : message.content
+                    .filter((c) => c.type === 'text')
+                    .map((c) => c.text ?? '')
+                    .join('');
+            if (message.role === 'user') {
+                return `${acc}\n\nHuman: ${content}`;
+            }
+            else if (message.role === 'assistant') {
+                return `${acc}\n\nAssistant: ${content}`;
+            }
+            return acc;
+        }, '');
+        prompt += '\n\nAssistant:';
+        // Get the "System" prompt input for Claude 3 models
+        const system = data.model.startsWith('claude-3') ? getSystemPrompt(inputs) : undefined;
+        let { maxTokens } = data;
+        const tokenizerInfo = {
+            node: context.node,
+            model,
+            endpoint: undefined,
+        };
+        const tokenCountEstimate = await context.tokenizer.getTokenCountForString(prompt, tokenizerInfo);
+        const modelInfo = anthropicModels[model] ?? {
+            maxTokens: Number.MAX_SAFE_INTEGER,
+            cost: {
+                prompt: 0,
+                completion: 0,
+            },
+        };
+        if (tokenCountEstimate >= modelInfo.maxTokens) {
+            throw new Error(`The model ${model} can only handle ${modelInfo.maxTokens} tokens, but ${tokenCountEstimate} were provided in the prompts alone.`);
+        }
+        if (tokenCountEstimate + maxTokens > modelInfo.maxTokens) {
+            const message = `The model can only handle a maximum of ${modelInfo.maxTokens} tokens, but the prompts and max tokens together exceed this limit. The max tokens has been reduced to ${modelInfo.maxTokens - tokenCountEstimate}.`;
+            addWarning(output, message);
+            maxTokens = Math.floor((modelInfo.maxTokens - tokenCountEstimate) * 0.95); // reduce max tokens by 5% to be safe, calculation is a little wrong.
+        }
+        try {
+            return await retry(async () => {
+                const completionOptions = {
+                    model,
+                    temperature: useTopP ? undefined : temperature,
+                    top_p: useTopP ? topP : undefined,
+                    max_tokens_to_sample: maxTokens ?? modelInfo.maxTokens,
+                    stop_sequences: stop ? [stop] : undefined,
+                    prompt,
+                };
+                const messageOptions = {
+                    model,
+                    temperature: useTopP ? undefined : temperature,
+                    top_p: useTopP ? topP : undefined,
+                    max_tokens: maxTokens ?? modelInfo.maxTokens,
+                    stop_sequences: stop ? [stop] : undefined,
+                    system: system,
+                    messages,
+                    tools: tools
+                        ? tools.map((tool) => ({ name: tool.name, description: tool.description, input_schema: tool.parameters }))
+                        : undefined,
+                };
+                const useMessageApi = model.startsWith('claude-3');
+                const cacheKey = JSON.stringify(useMessageApi ? messageOptions : completionOptions);
+                if (data.cache) {
+                    const cached = cache.get(cacheKey);
+                    if (cached) {
+                        return cached;
+                    }
+                }
+                const startTime = Date.now();
+                const apiKey = context.getPluginConfig('anthropicApiKey');
+                if (useMessageApi && data.enableToolUse) {
+                    // Streaming is not supported with tool usage.
+                    const response = await callMessageApi({
+                        apiKey: apiKey ?? '',
+                        ...messageOptions,
+                    });
+                    const { input_tokens: requestTokens, output_tokens: responseTokens } = response.usage;
+                    const responseText = response.content
+                        .map((c) => c.text)
+                        .filter(isNotNull)
+                        .join('');
+                    output['response'] = {
+                        type: 'string',
+                        value: responseText,
+                    };
+                    const functionCalls = response.content
+                        .filter((content) => content.name && content.id)
+                        .map((functionCall) => ({
+                        name: functionCall.name,
+                        arguments: functionCall.input, // Matches OpenAI ChatNode
+                        id: functionCall.id,
+                    }));
+                    if (functionCalls.length > 0) {
+                        output['function-calls'] = {
+                            type: 'object[]',
+                            value: functionCalls,
+                        };
+                    }
+                    output['all-messages'] = {
+                        type: 'chat-message[]',
+                        value: [
+                            ...rivetChatMessages,
+                            {
+                                type: 'assistant',
+                                message: responseText,
+                                function_call: functionCalls.length > 0
+                                    ? functionCalls.map((toolCall) => ({
+                                        name: toolCall.name,
+                                        arguments: JSON.stringify(toolCall.arguments),
+                                        id: toolCall.id,
+                                    }))[0]
+                                    : undefined,
+                                function_calls: functionCalls.map((toolCall) => ({
+                                    name: toolCall.name,
+                                    arguments: JSON.stringify(toolCall.arguments),
+                                    id: toolCall.id,
+                                })),
+                            },
+                        ],
+                    };
+                    output['requestTokens'] = { type: 'number', value: requestTokens ?? tokenCountEstimate };
+                    const responseTokenCount = responseTokens ?? context.tokenizer.getTokenCountForString(responseText, tokenizerInfo);
+                    output['responseTokens'] = { type: 'number', value: responseTokenCount };
+                }
+                else if (useMessageApi) {
+                    // Use the messages API for Claude 3 models
+                    const chunks = streamMessageApi({
+                        apiKey: apiKey ?? '',
+                        signal: context.signal,
+                        ...messageOptions,
+                    });
+                    // Process the response chunks and update the output
+                    const responseParts = [];
+                    let requestTokens = undefined, responseTokens = undefined;
+                    for await (const chunk of chunks) {
+                        let completion = '';
+                        if (chunk.type === 'content_block_start') {
+                            completion = chunk.content_block.text;
+                        }
+                        else if (chunk.type === 'content_block_delta') {
+                            completion = chunk.delta.text;
+                        }
+                        else if (chunk.type === 'message_start' && chunk.message?.usage?.input_tokens) {
+                            requestTokens = chunk.message.usage.input_tokens;
+                        }
+                        else if (chunk.type === 'message_delta' && chunk.delta?.usage?.output_tokens) {
+                            responseTokens = chunk.delta.usage.output_tokens;
+                        }
+                        if (!completion) {
+                            continue;
+                        }
+                        responseParts.push(completion);
+                        output['response'] = {
+                            type: 'string',
+                            value: responseParts.join('').trim(),
+                        };
+                        output['all-messages'] = {
+                            type: 'chat-message[]',
+                            value: [
+                                ...rivetChatMessages,
+                                {
+                                    type: 'assistant',
+                                    message: responseParts.join('').trim(),
+                                    function_call: undefined,
+                                    function_calls: undefined,
+                                },
+                            ],
+                        };
+                        context.onPartialOutputs?.(output);
+                    }
+                    if (responseParts.length === 0) {
+                        throw new Error('No response from Anthropic');
+                    }
+                    output['requestTokens'] = { type: 'number', value: requestTokens ?? tokenCountEstimate };
+                    const responseTokenCount = responseTokens ?? (await context.tokenizer.getTokenCountForString(responseParts.join(''), tokenizerInfo));
+                    output['responseTokens'] = { type: 'number', value: responseTokenCount };
+                }
+                else {
+                    // Use the normal chat completion method for non-Claude 3 models
+                    const chunks = streamChatCompletions({
+                        apiKey: apiKey ?? '',
+                        signal: context.signal,
+                        ...completionOptions,
+                    });
+                    // Process the response chunks and update the output
+                    const responseParts = [];
+                    for await (const chunk of chunks) {
+                        if (!chunk.completion) {
+                            continue;
+                        }
+                        responseParts.push(chunk.completion);
+                        output['response'] = {
+                            type: 'string',
+                            value: responseParts.join('').trim(),
+                        };
+                        context.onPartialOutputs?.(output);
+                    }
+                    if (responseParts.length === 0) {
+                        throw new Error('No response from Anthropic');
+                    }
+                    output['all-messages'] = {
+                        type: 'chat-message[]',
+                        value: [
+                            ...rivetChatMessages,
+                            {
+                                type: 'assistant',
+                                message: responseParts.join('').trim(),
+                                function_call: undefined,
+                                function_calls: undefined,
+                            },
+                        ],
+                    };
+                    output['requestTokens'] = { type: 'number', value: tokenCountEstimate };
+                    const responseTokenCount = await context.tokenizer.getTokenCountForString(responseParts.join(''), tokenizerInfo);
+                    output['responseTokens'] = { type: 'number', value: responseTokenCount };
+                }
+                const cost = getCostForTokens({
+                    requestTokens: output['requestTokens']?.value,
+                    responseTokens: output['responseTokens']?.value,
+                }, model);
+                if (cost != null) {
+                    output['cost'] = { type: 'number', value: cost };
+                }
+                const endTime = Date.now();
+                const duration = endTime - startTime;
+                output['duration'] = { type: 'number', value: duration };
+                Object.freeze(output);
+                cache.set(cacheKey, output);
+                return output;
+            }, {
+                forever: true,
+                retries: 10000,
+                maxRetryTime: 1000 * 60 * 5,
+                factor: 2.5,
+                minTimeout: 500,
+                maxTimeout: 5000,
+                randomize: true,
+                signal: context.signal,
+                onFailedAttempt(err) {
+                    context.trace(`ChatAnthropicNode failed, retrying: ${err.toString()}`);
+                    if (context.signal.aborted) {
+                        throw new Error('Aborted');
+                    }
+                    if (err instanceof AnthropicError) {
+                        if (err.response.status >= 400 && err.response.status < 500) {
+                            if (err.responseJson.error?.message) {
+                                throw new Error(err.responseJson.error.message);
+                            }
+                        }
+                    }
+                },
+            });
+        }
+        catch (error) {
+            context.trace(getError(error).stack ?? 'Missing stack');
+            throw new Error(`Error processing ChatAnthropicNode: ${error.message}`);
+        }
+    },
+};
+export const chatAnthropicNode = pluginNodeDefinition(ChatAnthropicNodeImpl, 'Chat');
+export function getSystemPrompt(inputs) {
+    const system = coerceTypeOptional(inputs['system'], 'string');
+    if (system) {
+        return system;
+    }
+    const prompt = inputs['prompt'];
+    if (prompt && prompt.type === 'chat-message[]') {
+        const systemMessage = prompt.value.find((message) => message.type === 'system');
+        if (systemMessage) {
+            if (typeof systemMessage.message === 'string') {
+                return systemMessage.message;
+            }
+            else if (Array.isArray(systemMessage.message)) {
+                return systemMessage.message.filter((p) => typeof p === 'string').join('');
+            }
+        }
+    }
+}
+function getChatMessages(inputs) {
+    const prompt = inputs['prompt'];
+    if (!prompt) {
+        throw new Error('Prompt is required');
+    }
+    const chatMessages = match(prompt)
+        .with({ type: 'chat-message' }, (p) => [p.value])
+        .with({ type: 'chat-message[]' }, (p) => p.value)
+        .with({ type: 'string' }, (p) => [{ type: 'user', message: p.value }])
+        .with({ type: 'string[]' }, (p) => p.value.map((v) => ({ type: 'user', message: v })))
+        .otherwise((p) => {
+        if (isArrayDataValue(p)) {
+            const stringValues = p.value.map((v) => coerceType({
+                type: getScalarTypeOf(p.type),
+                value: v,
+            }, 'string'));
+            return stringValues.filter((v) => v != null).map((v) => ({ type: 'user', message: v }));
+        }
+        const coercedMessage = coerceType(p, 'chat-message');
+        if (coercedMessage != null) {
+            return [coercedMessage];
+        }
+        const coercedString = coerceType(p, 'string');
+        return coercedString != null ? [{ type: 'user', message: coerceType(p, 'string') }] : [];
+    });
+    return chatMessages;
+}
+export async function chatMessagesToClaude3ChatMessages(chatMessages) {
+    const messages = (await Promise.all(chatMessages.map(chatMessageToClaude3ChatMessage))).filter(isNotNull);
+    // Combine sequential tool_result messages into a single user message with multiple content items
+    const combinedMessages = messages.reduce((acc, message) => {
+        if (message.role === 'user' &&
+            Array.isArray(message.content) &&
+            message.content.length === 1 &&
+            message.content[0].type === 'tool_result') {
+            const last = acc.at(-1);
+            if (last?.role === 'user' && Array.isArray(last.content) && last.content.every((c) => c.type === 'tool_result')) {
+                const mc = message.content;
+                const content = last.content.concat(mc);
+                return [...acc.slice(0, -1), { ...last, content }];
+            }
+        }
+        return [...acc, message];
+    }, []);
+    return combinedMessages;
+}
+async function chatMessageToClaude3ChatMessage(message) {
+    if (message.type === 'system') {
+        return undefined;
+    }
+    if (message.type === 'function') {
+        // Interpret function messages as user messages with tool_result content items (making Claude API more similar to OpenAI's)
+        const content = (Array.isArray(message.message) ? message.message : [message.message])
+            .map((m) => (typeof m === 'string' ? { type: 'text', text: m } : undefined))
+            .filter(isNotNull);
+        return {
+            role: 'user',
+            content: [
+                {
+                    type: 'tool_result',
+                    tool_use_id: message.name,
+                    content: content.length === 1 ? content[0].text : content,
+                },
+            ],
+        };
+    }
+    const content = Array.isArray(message.message)
+        ? await Promise.all(message.message.map(chatMessageContentToClaude3ChatMessage))
+        : [await chatMessageContentToClaude3ChatMessage(message.message)];
+    if (message.type === 'assistant' && message.function_calls) {
+        content.push(...message.function_calls.map((fc) => ({
+            type: 'tool_use',
+            id: fc.id,
+            name: fc.name,
+            input: JSON.parse(fc.arguments),
+        })));
+    }
+    else if (message.type === 'assistant' && message.function_call) {
+        content.push({
+            type: 'tool_use',
+            id: message.function_call.id,
+            name: message.function_call.name,
+            input: JSON.parse(message.function_call.arguments),
+        });
+    }
+    return {
+        role: message.type,
+        content,
+    };
+}
+async function chatMessageContentToClaude3ChatMessage(content) {
+    if (typeof content === 'string') {
+        return {
+            type: 'text',
+            text: content,
+        };
+    }
+    switch (content.type) {
+        case 'image':
+            return {
+                type: 'image',
+                source: {
+                    type: 'base64',
+                    media_type: content.mediaType,
+                    data: (await uint8ArrayToBase64(content.data)) ?? '',
+                },
+            };
+        case 'url':
+            throw new Error('unable to convert urls for Claude');
+        default:
+            assertNever(content);
+    }
+}
+function getCostForTokens(tokenCounts, model) {
+    const modelInfo = anthropicModels[model];
+    if (modelInfo == null) {
+        return undefined;
+    }
+    return modelInfo.cost.prompt * tokenCounts.requestTokens + modelInfo.cost.completion * tokenCounts.responseTokens;
+}

package/dist/esm/plugins/anthropic/plugin.js

@@ -0,0 +1,18 @@
+import {} from '../../index.js';
+import { chatAnthropicNode } from './nodes/ChatAnthropicNode.js';
+export const anthropicPlugin = {
+    id: 'anthropic',
+    name: 'Anthropic',
+    register: (register) => {
+        register(chatAnthropicNode);
+    },
+    configSpec: {
+        anthropicApiKey: {
+            type: 'string',
+            label: 'Anthropic API Key',
+            description: 'The API key for the Anthropic service.',
+            pullEnvironmentVariable: 'ANTHROPIC_API_KEY',
+            helperText: 'You may also set the ANTHROPIC_API_KEY environment variable.',
+        },
+    },
+};