@lobehub/chat 0.144.0 → 0.144.1

package/CHANGELOG.md

@@ -2,6 +2,31 @@
 
 # Changelog
 
+### [Version 0.144.1](https://github.com/lobehub/lobe-chat/compare/v0.144.0...v0.144.1)
+
+<sup>Released on **2024-03-29**</sup>
+
+#### 🐛 Bug Fixes
+
+- **ollama**: Suppport vision for LLaVA models.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### What's fixed
+
+- **ollama**: Suppport vision for LLaVA models, closes [#1791](https://github.com/lobehub/lobe-chat/issues/1791) ([e2d3de6](https://github.com/lobehub/lobe-chat/commit/e2d3de6))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
 ## [Version 0.144.0](https://github.com/lobehub/lobe-chat/compare/v0.143.0...v0.144.0)
 
 <sup>Released on **2024-03-29**</sup>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "0.144.0",
+  "version": "0.144.1",
   "description": "Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.",
   "keywords": [
     "framework",

package/src/config/modelProviders/ollama.ts

@@ -135,7 +135,7 @@ const Ollama: ModelProviderCard = {
       hidden: true,
       id: 'llava',
       tokens: 4000,
-      vision: false,
+      vision: true,
     },
     {
       displayName: 'LLaVA 13B',
@@ -143,7 +143,7 @@ const Ollama: ModelProviderCard = {
       hidden: true,
       id: 'llava:13b',
       tokens: 4000,
-      vision: false,
+      vision: true,
     },
     {
       displayName: 'LLaVA 34B',
@@ -151,7 +151,7 @@ const Ollama: ModelProviderCard = {
       hidden: true,
       id: 'llava:34b',
       tokens: 4000,
-      vision: false,
+      vision: true,
     },
   ],
   id: 'ollama',

package/src/libs/agent-runtime/ollama/index.test.ts

@@ -2,7 +2,7 @@
 import OpenAI from 'openai';
 import { Mock, afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
-import { ChatStreamCallbacks } from '@/libs/agent-runtime';
+import { ChatStreamCallbacks, OpenAIChatMessage } from '@/libs/agent-runtime';
 
 import * as debugStreamModule from '../utils/debugStream';
 import { LobeOllamaAI } from './index';
@@ -317,4 +317,49 @@ describe('LobeOllamaAI', () => {
       });
     });
   });
+
+  describe('private method', () => {
+    describe('convertContentToOllamaMessage', () => {
+      it('should format message array content of UserMessageContentPart to match ollama api', () => {
+        const message: OpenAIChatMessage = {
+          role: 'user',
+          content: [
+            {
+              type: 'text',
+              text: 'Hello',
+            },
+            {
+              type: 'image_url',
+              image_url: {
+                detail: 'auto',
+                url: 'data:image/png;base64,iVBO...',
+              },
+            },
+          ],
+        };
+
+        const ollamaMessage = instance['convertContentToOllamaMessage'](message);
+
+        expect(ollamaMessage).toEqual({
+          role: 'user',
+          content: 'Hello',
+          images: ['iVBO...'],
+        });
+      });
+
+      it('should not affect string type message content', () => {
+        const message: OpenAIChatMessage = {
+          role: 'user',
+          content: 'Hello',
+        };
+
+        const ollamaMessage = instance['convertContentToOllamaMessage'](message);
+
+        expect(ollamaMessage).toEqual({
+          role: 'user',
+          content: 'Hello',
+        });
+      });
+    });
+  });
 });

package/src/libs/agent-runtime/ollama/index.ts

@@ -1,6 +1,8 @@
 import { OpenAIStream, StreamingTextResponse } from 'ai';
 import OpenAI, { ClientOptions } from 'openai';
 
+import { OllamaChatMessage, OpenAIChatMessage } from '@/libs/agent-runtime';
+
 import { LobeRuntimeAI } from '../BaseAI';
 import { AgentRuntimeErrorType } from '../error';
 import { ChatCompetitionOptions, ChatStreamPayload, ModelProvider } from '../types';
@@ -8,6 +10,7 @@ import { AgentRuntimeError } from '../utils/createError';
 import { debugStream } from '../utils/debugStream';
 import { desensitizeUrl } from '../utils/desensitizeUrl';
 import { handleOpenAIError } from '../utils/handleOpenAIError';
+import { parseDataUri } from '../utils/uriParser';
 
 const DEFAULT_BASE_URL = 'http://127.0.0.1:11434/v1';
 
@@ -25,6 +28,8 @@ export class LobeOllamaAI implements LobeRuntimeAI {
 
   async chat(payload: ChatStreamPayload, options?: ChatCompetitionOptions) {
     try {
+      payload.messages = this.buildOllamaMessages(payload.messages);
+
       const response = await this.client.chat.completions.create(
         payload as unknown as OpenAI.ChatCompletionCreateParamsStreaming,
       );
@@ -73,6 +78,41 @@ export class LobeOllamaAI implements LobeRuntimeAI {
       });
     }
   }
+
+  private buildOllamaMessages(messages: OpenAIChatMessage[]) {
+    return messages.map((message) => this.convertContentToOllamaMessage(message));
+  }
+
+  private convertContentToOllamaMessage = (message: OpenAIChatMessage) => {
+    if (typeof message.content === 'string') {
+      return message;
+    }
+
+    const ollamaMessage: OllamaChatMessage = {
+      content: '',
+      role: message.role,
+    };
+
+    for (const content of message.content) {
+      switch (content.type) {
+        case 'text': {
+          // keep latest text input
+          ollamaMessage.content = content.text;
+          break;
+        }
+        case 'image_url': {
+          const { base64 } = parseDataUri(content.image_url.url);
+          if (base64) {
+            ollamaMessage.images ??= [];
+            ollamaMessage.images.push(base64);
+          }
+          break;
+        }
+      }
+    }
+
+    return ollamaMessage;
+  };
 }
 
 export default LobeOllamaAI;

package/src/libs/agent-runtime/types/chat.ts

@@ -127,3 +127,10 @@ export interface ChatCompletionTool {
 }
 
 export type ChatStreamCallbacks = OpenAIStreamCallbacks;
+
+export interface OllamaChatMessage extends OpenAIChatMessage {
+  /**
+   * @description images for ollama vision models (https://ollama.com/blog/vision-models)
+   */
+  images?: string[];
+}