@lobehub/chat 0.144.0 → 0.145.0

package/src/app/api/chat/[provider]/agentRuntime.ts

@@ -21,6 +21,7 @@ import {
   LobeOpenRouterAI,
   LobePerplexityAI,
   LobeRuntimeAI,
+  LobeTogetherAI,
   LobeZeroOneAI,
   LobeZhipuAI,
   ModelProvider,
@@ -181,6 +182,11 @@ class AgentRuntime {
         break;
       }
 
+      case ModelProvider.TogetherAI: {
+        runtimeModel = this.initTogetherAI(payload);
+        break;
+      }
+
       case ModelProvider.ZeroOne: {
         runtimeModel = this.initZeroOne(payload);
         break;
@@ -301,13 +307,19 @@ class AgentRuntime {
     return new LobeOpenRouterAI({ apiKey });
   }
 
+  private static initTogetherAI(payload: JWTPayload) {
+    const { TOGETHERAI_API_KEY } = getServerConfig();
+    const apiKey = apiKeyManager.pick(payload?.apiKey || TOGETHERAI_API_KEY);
+
+    return new LobeTogetherAI({ apiKey });
+  }
+
   private static initZeroOne(payload: JWTPayload) {
     const { ZEROONE_API_KEY } = getServerConfig();
     const apiKey = apiKeyManager.pick(payload?.apiKey || ZEROONE_API_KEY);
 
     return new LobeZeroOneAI({ apiKey });
   }
-
 }
 
 export default AgentRuntime;

package/src/app/api/config/route.ts

@@ -24,6 +24,7 @@ export const GET = async () => {
     ENABLED_MISTRAL,
     ENABLED_OPENROUTER,
     ENABLED_ZEROONE,
+    ENABLED_TOGETHERAI,
     DEFAULT_AGENT_CONFIG,
     OLLAMA_CUSTOM_MODELS,
     OPENROUTER_CUSTOM_MODELS,
@@ -46,6 +47,7 @@ export const GET = async () => {
       ollama: { customModelName: OLLAMA_CUSTOM_MODELS, enabled: ENABLE_OLLAMA },
       openrouter: { customModelName: OPENROUTER_CUSTOM_MODELS, enabled: ENABLED_OPENROUTER },
       perplexity: { enabled: ENABLED_PERPLEXITY },
+      togetherai: { enabled: ENABLED_TOGETHERAI },
       zeroone: { enabled: ENABLED_ZEROONE },
       zhipu: { enabled: ENABLED_ZHIPU },
     },

package/src/app/api/errorResponse.test.ts

@@ -121,6 +121,11 @@ describe('createErrorResponse', () => {
       expect(response.status).toBe(481);
     });
 
+    it('returns a 484 status for TogetherAIBizError error type', () => {
+      const errorType = AgentRuntimeErrorType.TogetherAIBizError;
+      const response = createErrorResponse(errorType);
+      expect(response.status).toBe(484);
+    });
   });
 
   // 测试状态码不在200-599范围内的情况

package/src/app/api/errorResponse.ts

@@ -59,6 +59,9 @@ const getStatus = (errorType: ILobeAgentRuntimeErrorType | ErrorType) => {
     case AgentRuntimeErrorType.ZeroOneBizError: {
       return 483;
     }
+    case AgentRuntimeErrorType.TogetherAIBizError: {
+      return 484;
+    }
   }
   return errorType as number;
 };

package/src/app/settings/llm/Google/index.tsx

@@ -1,5 +1,5 @@
-import { Google, Gemini } from '@lobehub/icons';
-import { Input, Divider } from 'antd';
+import { Gemini, Google } from '@lobehub/icons';
+import { Divider, Input } from 'antd';
 import { memo } from 'react';
 import { useTranslation } from 'react-i18next';
 import { Flexbox } from 'react-layout-kit';

package/src/app/settings/llm/TogetherAI/index.tsx

@@ -0,0 +1,66 @@
+import { Together } from '@lobehub/icons';
+import { Input } from 'antd';
+import { useTheme } from 'antd-style';
+import { memo } from 'react';
+import { useTranslation } from 'react-i18next';
+
+import { ModelProvider } from '@/libs/agent-runtime';
+
+import Checker from '../components/Checker';
+import ProviderConfig from '../components/ProviderConfig';
+import { LLMProviderApiTokenKey, LLMProviderConfigKey } from '../const';
+
+const providerKey = 'togetherai';
+
+const TogetherAIProvider = memo(() => {
+  const { t } = useTranslation('setting');
+
+  const theme = useTheme();
+
+  return (
+    <ProviderConfig
+      configItems={[
+        {
+          children: (
+            <Input.Password
+              autoComplete={'new-password'}
+              placeholder={t('llm.TogetherAI.token.placeholder')}
+            />
+          ),
+          desc: t('llm.TogetherAI.token.desc'),
+          label: t('llm.TogetherAI.token.title'),
+          name: [LLMProviderConfigKey, providerKey, LLMProviderApiTokenKey],
+        },
+        {
+          children: (
+            <Input.TextArea
+              allowClear
+              placeholder={t('llm.TogetherAI.customModelName.placeholder')}
+              style={{ height: 100 }}
+            />
+          ),
+          desc: t('llm.TogetherAI.customModelName.desc'),
+          label: t('llm.TogetherAI.customModelName.title'),
+          name: [LLMProviderConfigKey, providerKey, 'customModelName'],
+        },
+        {
+          children: (
+            <Checker model={'togethercomputer/alpaca-7b'} provider={ModelProvider.TogetherAI} />
+          ),
+          desc: t('llm.checker.desc'),
+          label: t('llm.checker.title'),
+          minWidth: '100%',
+        },
+      ]}
+      provider={providerKey}
+      title={
+        <Together.Combine
+          color={theme.isDarkMode ? theme.colorText : Together.colorPrimary}
+          size={24}
+        />
+      }
+    />
+  );
+});
+
+export default TogetherAIProvider;

package/src/app/settings/llm/index.tsx

@@ -18,6 +18,7 @@ import Ollama from './Ollama';
 import OpenAI from './OpenAI';
 import OpenRouter from './OpenRouter';
 import Perplexity from './Perplexity';
+import TogetherAI from './TogetherAI';
 import ZeroOne from './ZeroOne';
 import Zhipu from './Zhipu';
 
@@ -40,6 +41,7 @@ export default memo<{ showOllama: boolean }>(({ showOllama }) => {
       <Moonshot />
       <ZeroOne />
       <Zhipu />
+      <TogetherAI />
       <Footer>
         <Trans i18nKey="llm.waitingForMore" ns={'setting'}>
           更多模型正在

package/src/components/ModelProviderIcon/index.tsx

@@ -10,6 +10,7 @@ import {
   OpenAI,
   OpenRouter,
   Perplexity,
+  Together,
   ZeroOne,
   Zhipu,
 } from '@lobehub/icons';
@@ -84,6 +85,10 @@ const ModelProviderIcon = memo<ModelProviderIconProps>(({ provider }) => {
       return <ZeroOne size={20} />;
     }
 
+    case ModelProvider.TogetherAI: {
+      return <Together size={20} />;
+    }
+
     default: {
       return null;
     }

package/src/config/modelProviders/index.ts

@@ -10,6 +10,7 @@ import OllamaProvider from './ollama';
 import OpenAIProvider from './openai';
 import OpenRouterProvider from './openrouter';
 import PerplexityProvider from './perplexity';
+import TogetherAIProvider from './togetherai';
 import ZeroOneProvider from './zeroone';
 import ZhiPuProvider from './zhipu';
 
@@ -23,6 +24,7 @@ export const LOBE_DEFAULT_MODEL_LIST: ChatModelCard[] = [
   MoonshotProvider.chatModels,
   OllamaProvider.chatModels,
   OpenRouterProvider.chatModels,
+  TogetherAIProvider.chatModels,
   PerplexityProvider.chatModels,
   AnthropicProvider.chatModels,
   ZeroOneProvider.chatModels,
@@ -38,5 +40,6 @@ export { default as OllamaProvider } from './ollama';
 export { default as OpenAIProvider } from './openai';
 export { default as OpenRouterProvider } from './openrouter';
 export { default as PerplexityProvider } from './perplexity';
+export { default as TogetherAIProvider } from './togetherai';
 export { default as ZeroOneProvider } from './zeroone';
 export { default as ZhiPuProvider } from './zhipu';

package/src/config/modelProviders/ollama.ts

@@ -135,7 +135,7 @@ const Ollama: ModelProviderCard = {
       hidden: true,
       id: 'llava',
       tokens: 4000,
-      vision: false,
+      vision: true,
     },
     {
       displayName: 'LLaVA 13B',
@@ -143,7 +143,7 @@ const Ollama: ModelProviderCard = {
       hidden: true,
       id: 'llava:13b',
       tokens: 4000,
-      vision: false,
+      vision: true,
     },
     {
       displayName: 'LLaVA 34B',
@@ -151,7 +151,7 @@ const Ollama: ModelProviderCard = {
       hidden: true,
       id: 'llava:34b',
       tokens: 4000,
-      vision: false,
+      vision: true,
     },
   ],
   id: 'ollama',

package/src/config/modelProviders/togetherai.ts

@@ -0,0 +1,86 @@
+import { ModelProviderCard } from '@/types/llm';
+
+const TogetherAI: ModelProviderCard = {
+  chatModels: [
+    {
+      displayName: 'Deepseek Coder Instruct (33B)',
+      functionCall: false,
+      id: 'deepseek-ai/deepseek-coder-33b-instruct',
+      tokens: 16_384,
+      vision: false,
+    },
+    {
+      displayName: 'Phind Code LLaMA v2 (34B)',
+      functionCall: false,
+      id: 'Phind/Phind-CodeLlama-34B-v2',
+      tokens: 16_384,
+      vision: false,
+    },
+    {
+      displayName: 'Gemma Instruct (2B)',
+      functionCall: false,
+      id: 'google/gemma-2b-it',
+      tokens: 8192,
+      vision: false,
+    },
+    {
+      displayName: 'LLaMA-2 Chat (13B)',
+      functionCall: false,
+      id: 'meta-llama/Llama-2-13b-chat-hf',
+      tokens: 4096,
+      vision: false,
+    },
+    {
+      displayName: '01-ai Yi Chat (34B)',
+      functionCall: false,
+      id: 'zero-one-ai/Yi-34B-Chat',
+      tokens: 4096,
+      vision: false,
+    },
+    {
+      displayName: 'Mixtral-8x7B Instruct (46.7B)',
+      functionCall: false,
+      id: 'mistralai/Mixtral-8x7B-Instruct-v0.1',
+      tokens: 32_768,
+      vision: false,
+    },
+    {
+      displayName: 'Nous Hermes 2 - Mixtral 8x7B-DPO (46.7B)',
+      functionCall: false,
+      id: 'NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO',
+      tokens: 32_768,
+      vision: false,
+    },
+    {
+      displayName: 'Nous Hermes-2 Yi (34B)',
+      functionCall: false,
+      id: 'NousResearch/Nous-Hermes-2-Yi-34B',
+      tokens: 4096,
+      vision: false,
+    },
+    {
+      displayName: 'Qwen 1.5 Chat (7B)',
+      functionCall: false,
+      id: 'Qwen/Qwen1.5-7B-Chat',
+      tokens: 32_768,
+      vision: false,
+    },
+    {
+      displayName: 'Qwen 1.5 Chat (14B)',
+      functionCall: false,
+      id: 'Qwen/Qwen1.5-14B-Chat',
+      tokens: 32_768,
+      vision: false,
+    },
+    {
+      displayName: 'Qwen 1.5 Chat (72B)',
+      functionCall: false,
+      id: 'Qwen/Qwen1.5-72B-Chat',
+      tokens: 4096,
+      vision: false,
+    },
+  ],
+  id: 'togetherai',
+};
+
+export default TogetherAI;

package/src/config/modelProviders/zeroone.ts

@@ -9,7 +9,8 @@ const ZeroOne: ModelProviderCard = {
       tokens: 4000,
     },
     {
-      description: '支持通用图片问答、图表理解、OCR、视觉推理，能处理高分辨率（1024*1024）的图像，能在复杂视觉任务上提供优秀性能，同时支持多种语言。',
+      description:
+        '支持通用图片问答、图表理解、OCR、视觉推理，能处理高分辨率（1024*1024）的图像，能在复杂视觉任务上提供优秀性能，同时支持多种语言。',
       displayName: 'YI Vision Plus',
       id: 'yi-vl-plus',
       tokens: 4000,

package/src/config/server/provider.ts

@@ -50,6 +50,9 @@ declare global {
       // ZeroOne Provider
       ZEROONE_API_KEY?: string;
 
+      // TogetherAI Provider
+      TOGETHERAI_API_KEY?: string;
+
       // AWS Credentials
       AWS_REGION?: string;
       AWS_ACCESS_KEY_ID?: string;
@@ -86,6 +89,8 @@ export const getProviderConfig = () => {
 
   const ZEROONE_API_KEY = process.env.ZEROONE_API_KEY || '';
 
+  const TOGETHERAI_API_KEY = process.env.TOGETHERAI_API_KEY || '';
+
   // region format: iad1,sfo1
   let regions: string[] = [];
   if (process.env.OPENAI_FUNCTION_REGIONS) {
@@ -121,6 +126,9 @@ export const getProviderConfig = () => {
     OPENROUTER_API_KEY,
     OPENROUTER_CUSTOM_MODELS: process.env.OPENROUTER_CUSTOM_MODELS,
 
+    ENABLED_TOGETHERAI: !!TOGETHERAI_API_KEY,
+    TOGETHERAI_API_KEY,
+
     ENABLED_MOONSHOT: !!MOONSHOT_API_KEY,
     MOONSHOT_API_KEY,
     MOONSHOT_PROXY_URL: process.env.MOONSHOT_PROXY_URL,

package/src/const/settings.ts

@@ -96,6 +96,10 @@ export const DEFAULT_LLM_CONFIG: GlobalLLMConfig = {
     apiKey: '',
     enabled: false,
   },
+  togetherai: {
+    apiKey: '',
+    enabled: false,
+  },
   zeroone: {
     apiKey: '',
     enabled: false,

package/src/features/Conversation/Error/APIKeyForm/TogetherAI.tsx

@@ -0,0 +1,40 @@
+import { Together } from '@lobehub/icons';
+import { Input } from 'antd';
+import { memo } from 'react';
+import { useTranslation } from 'react-i18next';
+
+import { ModelProvider } from '@/libs/agent-runtime';
+import { useGlobalStore } from '@/store/global';
+import { modelProviderSelectors } from '@/store/global/selectors';
+
+import { FormAction } from '../style';
+
+const TogetherAIForm = memo(() => {
+  const { t } = useTranslation('error');
+  // const [showProxy, setShow] = useState(false);
+
+  const [apiKey, setConfig] = useGlobalStore((s) => [
+    modelProviderSelectors.togetheraiAPIKey(s),
+    s.setModelProviderConfig,
+  ]);
+
+  return (
+    <FormAction
+      avatar={<Together size={56} />}
+      description={t('unlock.apikey.TogetherAI.description')}
+      title={t('unlock.apikey.TogetherAI.title')}
+    >
+      <Input.Password
+        autoComplete={'new-password'}
+        onChange={(e) => {
+          setConfig(ModelProvider.TogetherAI, { apiKey: e.target.value });
+        }}
+        placeholder={'*********************************'}
+        type={'block'}
+        value={apiKey}
+      />
+    </FormAction>
+  );
+});
+
+export default TogetherAIForm;

package/src/features/Conversation/Error/APIKeyForm/index.tsx

@@ -15,6 +15,7 @@ import MoonshotForm from './Moonshot';
 import OpenAIForm from './OpenAI';
 import OpenRouterForm from './OpenRouter';
 import PerplexityForm from './Perplexity';
+import TogetherAIForm from './TogetherAI';
 import ZeroOneForm from './ZeroOne';
 import ZhipuForm from './Zhipu';
 
@@ -66,6 +67,10 @@ const APIKeyForm = memo<APIKeyFormProps>(({ id, provider }) => {
         return <OpenRouterForm />;
       }
 
+      case ModelProvider.TogetherAI: {
+        return <TogetherAIForm />;
+      }
+
       case ModelProvider.ZeroOne: {
         return <ZeroOneForm />;
       }

package/src/features/Conversation/Error/index.tsx

@@ -76,6 +76,7 @@ const ErrorMessageExtra = memo<{ data: ChatMessage }>(({ data }) => {
     case AgentRuntimeErrorType.InvalidAnthropicAPIKey:
     case AgentRuntimeErrorType.InvalidGroqAPIKey:
     case AgentRuntimeErrorType.InvalidOpenRouterAPIKey:
+    case AgentRuntimeErrorType.InvalidTogetherAIAPIKey:
     case AgentRuntimeErrorType.InvalidZeroOneAPIKey:
     case AgentRuntimeErrorType.NoOpenAIAPIKey: {
       return <InvalidAPIKey id={data.id} provider={data.error?.body?.provider} />;

package/src/libs/agent-runtime/error.ts

@@ -43,6 +43,9 @@ export const AgentRuntimeErrorType = {
 
   InvalidOpenRouterAPIKey: 'InvalidOpenRouterAPIKey',
   OpenRouterBizError: 'OpenRouterBizError',
+
+  InvalidTogetherAIAPIKey: 'InvalidTogetherAIAPIKey',
+  TogetherAIBizError: 'TogetherAIBizError',
 } as const;
 
 export type ILobeAgentRuntimeErrorType =

package/src/libs/agent-runtime/index.ts

@@ -11,6 +11,7 @@ export { LobeOllamaAI } from './ollama';
 export { LobeOpenAI } from './openai';
 export { LobeOpenRouterAI } from './openrouter';
 export { LobePerplexityAI } from './perplexity';
+export { LobeTogetherAI } from './togetherai';
 export * from './types';
 export { AgentRuntimeError } from './utils/createError';
 export { LobeZeroOneAI } from './zeroone';

package/src/libs/agent-runtime/ollama/index.test.ts

@@ -2,7 +2,7 @@
 import OpenAI from 'openai';
 import { Mock, afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
-import { ChatStreamCallbacks } from '@/libs/agent-runtime';
+import { ChatStreamCallbacks, OpenAIChatMessage } from '@/libs/agent-runtime';
 
 import * as debugStreamModule from '../utils/debugStream';
 import { LobeOllamaAI } from './index';
@@ -317,4 +317,49 @@ describe('LobeOllamaAI', () => {
       });
     });
   });
+
+  describe('private method', () => {
+    describe('convertContentToOllamaMessage', () => {
+      it('should format message array content of UserMessageContentPart to match ollama api', () => {
+        const message: OpenAIChatMessage = {
+          role: 'user',
+          content: [
+            {
+              type: 'text',
+              text: 'Hello',
+            },
+            {
+              type: 'image_url',
+              image_url: {
+                detail: 'auto',
+                url: 'data:image/png;base64,iVBO...',
+              },
+            },
+          ],
+        };
+
+        const ollamaMessage = instance['convertContentToOllamaMessage'](message);
+
+        expect(ollamaMessage).toEqual({
+          role: 'user',
+          content: 'Hello',
+          images: ['iVBO...'],
+        });
+      });
+
+      it('should not affect string type message content', () => {
+        const message: OpenAIChatMessage = {
+          role: 'user',
+          content: 'Hello',
+        };
+
+        const ollamaMessage = instance['convertContentToOllamaMessage'](message);
+
+        expect(ollamaMessage).toEqual({
+          role: 'user',
+          content: 'Hello',
+        });
+      });
+    });
+  });
 });

package/src/libs/agent-runtime/ollama/index.ts

@@ -1,6 +1,8 @@
 import { OpenAIStream, StreamingTextResponse } from 'ai';
 import OpenAI, { ClientOptions } from 'openai';
 
+import { OllamaChatMessage, OpenAIChatMessage } from '@/libs/agent-runtime';
+
 import { LobeRuntimeAI } from '../BaseAI';
 import { AgentRuntimeErrorType } from '../error';
 import { ChatCompetitionOptions, ChatStreamPayload, ModelProvider } from '../types';
@@ -8,6 +10,7 @@ import { AgentRuntimeError } from '../utils/createError';
 import { debugStream } from '../utils/debugStream';
 import { desensitizeUrl } from '../utils/desensitizeUrl';
 import { handleOpenAIError } from '../utils/handleOpenAIError';
+import { parseDataUri } from '../utils/uriParser';
 
 const DEFAULT_BASE_URL = 'http://127.0.0.1:11434/v1';
 
@@ -25,6 +28,8 @@ export class LobeOllamaAI implements LobeRuntimeAI {
 
   async chat(payload: ChatStreamPayload, options?: ChatCompetitionOptions) {
     try {
+      payload.messages = this.buildOllamaMessages(payload.messages);
+
       const response = await this.client.chat.completions.create(
         payload as unknown as OpenAI.ChatCompletionCreateParamsStreaming,
       );
@@ -73,6 +78,41 @@ export class LobeOllamaAI implements LobeRuntimeAI {
       });
     }
   }
+
+  private buildOllamaMessages(messages: OpenAIChatMessage[]) {
+    return messages.map((message) => this.convertContentToOllamaMessage(message));
+  }
+
+  private convertContentToOllamaMessage = (message: OpenAIChatMessage) => {
+    if (typeof message.content === 'string') {
+      return message;
+    }
+
+    const ollamaMessage: OllamaChatMessage = {
+      content: '',
+      role: message.role,
+    };
+
+    for (const content of message.content) {
+      switch (content.type) {
+        case 'text': {
+          // keep latest text input
+          ollamaMessage.content = content.text;
+          break;
+        }
+        case 'image_url': {
+          const { base64 } = parseDataUri(content.image_url.url);
+          if (base64) {
+            ollamaMessage.images ??= [];
+            ollamaMessage.images.push(base64);
+          }
+          break;
+        }
+      }
+    }
+
+    return ollamaMessage;
+  };
 }
 
 export default LobeOllamaAI;