@lobehub/chat 1.116.4 → 1.117.1

package/packages/model-runtime/src/utils/streams/openai/openai.test.ts

@@ -2271,4 +2271,45 @@ describe('OpenAIStream', () => {
       );
     });
   });
+
+  it('should handle base64_image in delta.images (image_url shape)', async () => {
+    const base64 =
+      'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==';
+
+    const mockOpenAIStream = new ReadableStream({
+      start(controller) {
+        controller.enqueue({
+          choices: [
+            {
+              delta: {
+                images: [
+                  {
+                    type: 'image_url',
+                    image_url: { url: base64 },
+                    index: 0,
+                  },
+                ],
+              },
+              index: 0,
+            },
+          ],
+          id: '6',
+        });
+
+        controller.close();
+      },
+    });
+
+    const protocolStream = OpenAIStream(mockOpenAIStream);
+
+    const decoder = new TextDecoder();
+    const chunks = [];
+
+    // @ts-ignore
+    for await (const chunk of protocolStream) {
+      chunks.push(decoder.decode(chunk, { stream: true }));
+    }
+
+    expect(chunks).toEqual(['id: 6\n', 'event: base64_image\n', `data: "${base64}"\n\n`]);
+  });
 });

package/packages/model-runtime/src/utils/streams/openai/openai.ts

@@ -96,6 +96,36 @@ const transformOpenAIStream = (
       }
     }
 
+    // Handle image preview chunks (e.g. Gemini 2.5 flash image preview)
+    // Example shape:
+    // choices[0].delta.images = [{ type: 'image_url', image_url: { url: 'data:image/png;base64,...' }, index: 0 }]
+    if (
+      (item as any).delta &&
+      Array.isArray((item as any).delta.images) &&
+      (item as any).delta.images.length > 0
+    ) {
+      const images = (item as any).delta.images as any[];
+
+      return images
+        .map((img) => {
+          // support multiple possible shapes for the url
+          const url =
+            img?.image_url?.url ||
+            img?.image_url?.image_url?.url ||
+            img?.url ||
+            (typeof img === 'string' ? img : undefined);
+
+          if (!url) return null;
+
+          return {
+            data: url,
+            id: chunk.id,
+            type: 'base64_image',
+          } as StreamProtocolChunk;
+        })
+        .filter(Boolean) as StreamProtocolChunk[];
+    }
+
     // 给定结束原因
     if (item.finish_reason) {
       // one-api 的流式接口，会出现既有 finish_reason ，也有 content 的情况
@@ -192,11 +222,11 @@ const transformOpenAIStream = (
         if ('content' in item.delta && Array.isArray(item.delta.content)) {
           return item.delta.content
             .filter((block: any) => block.type === 'thinking' && Array.isArray(block.thinking))
-            .map((block: any) => 
+            .map((block: any) =>
               block.thinking
                 .filter((thinkItem: any) => thinkItem.type === 'text' && thinkItem.text)
                 .map((thinkItem: any) => thinkItem.text)
-                .join('')
+                .join(''),
             )
             .join('');
         }
@@ -233,6 +263,12 @@ const transformOpenAIStream = (
           streamContext.thinkingInContent = false;
         }
 
+        // 如果 content 是空字符串但 chunk 带有 usage，则优先返回 usage（例如 Gemini image-preview 最终会在单独的 chunk 中返回 usage）
+        if (content === '' && chunk.usage) {
+          const usage = chunk.usage;
+          return { data: convertUsage(usage, provider), id: chunk.id, type: 'usage' };
+        }
+
         // 判断是否有 citations 内容，更新 returnedCitation 状态
         if (!streamContext?.returnedCitation) {
           const citations =

package/packages/model-runtime/src/utils/streams/protocol.test.ts

@@ -200,4 +200,36 @@ describe('createTokenSpeedCalculator', async () => {
     const results = await processChunk(transformer, chunks);
     expect(results).toHaveLength(chunks.length);
   });
+
+  it('should calculate token speed considering outputImageTokens when totalOutputTokens is missing', async () => {
+    const chunks = [
+      { data: '', id: 'chatcmpl-image-1', type: 'text' },
+      { data: 'hi', id: 'chatcmpl-image-1', type: 'text' },
+      { data: 'stop', id: 'chatcmpl-image-1', type: 'stop' },
+      {
+        data: {
+          inputTextTokens: 9,
+          outputTextTokens: 1,
+          outputImageTokens: 4,
+          totalInputTokens: 9,
+          // totalOutputTokens intentionally omitted to force summation path
+          totalTokens: 13,
+        },
+        id: 'chatcmpl-image-1',
+        type: 'usage',
+      },
+    ];
+
+    const transformer = createTokenSpeedCalculator((v) => v, { inputStartAt });
+    const results = await processChunk(transformer, chunks);
+
+    // should push an extra speed chunk
+    expect(results).toHaveLength(chunks.length + 1);
+    const speedChunk = results.slice(-1)[0];
+    expect(speedChunk.id).toBe('output_speed');
+    expect(speedChunk.type).toBe('speed');
+    // tps and ttft should be numeric (avoid flakiness if interval is 0ms)
+    expect(speedChunk.data.tps).not.toBeNaN();
+    expect(speedChunk.data.ttft).not.toBeNaN();
+  });
 });

package/packages/model-runtime/src/utils/streams/protocol.ts

@@ -364,10 +364,14 @@ export const createTokenSpeedCalculator = (
     }
     // if the chunk is the stop chunk, set as output finish
     if (inputStartAt && outputStartAt && chunk.type === 'usage') {
-      const totalOutputTokens = chunk.data?.totalOutputTokens || chunk.data?.outputTextTokens;
-      const reasoningTokens = chunk.data?.outputReasoningTokens || 0;
+      const totalOutputTokens =
+        chunk.data?.totalOutputTokens ??
+        (chunk.data?.outputTextTokens ?? 0) + (chunk.data?.outputImageTokens ?? 0);
+      const reasoningTokens = chunk.data?.outputReasoningTokens ?? 0;
       const outputTokens =
-        (outputThinking ?? false) ? totalOutputTokens : totalOutputTokens - reasoningTokens;
+        (outputThinking ?? false)
+          ? totalOutputTokens
+          : Math.max(0, totalOutputTokens - reasoningTokens);
       result.push({
         data: {
           tps: (outputTokens / (Date.now() - outputStartAt)) * 1000,

package/packages/model-runtime/src/utils/usageConverter.test.ts

@@ -290,4 +290,62 @@ describe('convertUsage', () => {
       totalTokens: 6550,
     });
   });
+
+  it('should handle output image tokens correctly', () => {
+    // Arrange
+    const usageWithImage = {
+      prompt_tokens: 100,
+      completion_tokens: 200,
+      completion_tokens_details: {
+        image_tokens: 60,
+        reasoning_tokens: 30,
+      },
+      total_tokens: 300,
+    } as OpenAI.Completions.CompletionUsage;
+
+    // Act
+    const result = convertUsage(usageWithImage);
+
+    // Assert
+    expect(result).toEqual({
+      inputTextTokens: 100,
+      totalInputTokens: 100,
+      totalOutputTokens: 200,
+      outputImageTokens: 60,
+      outputReasoningTokens: 30,
+      outputTextTokens: 110, // 200 - 60 - 30
+      totalTokens: 300,
+    });
+  });
+
+  it('should handle response output image tokens correctly for ResponseUsage', () => {
+    // Arrange
+    const responseUsage = {
+      input_tokens: 100,
+      input_tokens_details: {
+        cached_tokens: 0,
+      },
+      output_tokens: 200,
+      output_tokens_details: {
+        image_tokens: 60,
+        reasoning_tokens: 30,
+      },
+      total_tokens: 300,
+    } as OpenAI.Responses.ResponseUsage;
+
+    // Act
+    const result = convertResponseUsage(responseUsage);
+
+    // Assert
+    expect(result).toEqual({
+      inputTextTokens: 100,
+      inputCacheMissTokens: 100, // 100 - 0
+      totalInputTokens: 100,
+      totalOutputTokens: 200,
+      outputImageTokens: 60,
+      outputReasoningTokens: 30,
+      outputTextTokens: 170, // 200 - 30
+      totalTokens: 300,
+    });
+  });
 });

package/packages/model-runtime/src/utils/usageConverter.ts

@@ -20,12 +20,13 @@ export const convertUsage = (
   const totalOutputTokens = usage.completion_tokens;
   const outputReasoning = usage.completion_tokens_details?.reasoning_tokens || 0;
   const outputAudioTokens = usage.completion_tokens_details?.audio_tokens || 0;
+  const outputImageTokens = (usage.completion_tokens_details as any)?.image_tokens || 0;
 
   // XAI 的 completion_tokens 不包含 reasoning_tokens，需要特殊处理
   const outputTextTokens =
     provider === 'xai'
       ? totalOutputTokens - outputAudioTokens
-      : totalOutputTokens - outputReasoning - outputAudioTokens;
+      : totalOutputTokens - outputReasoning - outputAudioTokens - outputImageTokens;
 
   const totalTokens = inputCitationTokens + usage.total_tokens;
 
@@ -37,6 +38,7 @@ export const convertUsage = (
     inputCitationTokens: inputCitationTokens,
     inputTextTokens: inputTextTokens,
     outputAudioTokens: outputAudioTokens,
+    outputImageTokens: outputImageTokens,
     outputReasoningTokens: outputReasoning,
     outputTextTokens: outputTextTokens,
     rejectedPredictionTokens: usage.completion_tokens_details?.rejected_prediction_tokens,
@@ -75,6 +77,7 @@ export const convertResponseUsage = (usage: OpenAI.Responses.ResponseUsage): Mod
 
   // For ResponseUsage, outputTextTokens is totalOutputTokens minus reasoning, as no audio output tokens are specified.
   const outputTextTokens = totalOutputTokens - outputReasoningTokens;
+  const outputImageTokens = (usage.output_tokens_details as any)?.image_tokens || 0;
 
   // 3. Construct the comprehensive data object (matching ModelTokensUsage structure)
   const data = {
@@ -87,6 +90,7 @@ export const convertResponseUsage = (usage: OpenAI.Responses.ResponseUsage): Mod
     inputCitationTokens: undefined, // Not in ResponseUsage
     inputTextTokens: inputTextTokens,
     outputAudioTokens: undefined, // Not in ResponseUsage
+    outputImageTokens: outputImageTokens,
     outputReasoningTokens: outputReasoningTokens,
     outputTextTokens: outputTextTokens,
     rejectedPredictionTokens: undefined, // Not in ResponseUsage

package/packages/utils/vitest.config.mts

@@ -7,6 +7,7 @@ export default defineConfig({
       /* eslint-disable sort-keys-fix/sort-keys-fix */
       '@/types': resolve(__dirname, '../types/src'),
       '@/const': resolve(__dirname, '../const/src'),
+      '@/libs/model-runtime': resolve(__dirname, '../model-runtime/src'),
       '@': resolve(__dirname, '../../src'),
       /* eslint-enable */
     },

package/src/components/ChatItem/ChatItem.tsx

@@ -0,0 +1,183 @@
+'use client';
+
+import { useResponsive } from 'antd-style';
+import { memo, useEffect, useRef, useState } from 'react';
+import { Flexbox } from 'react-layout-kit';
+
+import Actions from './components/Actions';
+import Avatar from './components/Avatar';
+import BorderSpacing from './components/BorderSpacing';
+import ErrorContent from './components/ErrorContent';
+import MessageContent from './components/MessageContent';
+import Title from './components/Title';
+import { useStyles } from './style';
+import type { ChatItemProps } from './type';
+
+const MOBILE_AVATAR_SIZE = 32;
+
+const ChatItem = memo<ChatItemProps>(
+  ({
+    avatarAddon,
+    onAvatarClick,
+    avatarProps,
+    actions,
+    className,
+    primary,
+    loading,
+    message,
+    placeholderMessage = '...',
+    placement = 'left',
+    variant = 'bubble',
+    avatar,
+    error,
+    showTitle,
+    time,
+    editing,
+    onChange,
+    onEditingChange,
+    messageExtra,
+    renderMessage,
+    text,
+    errorMessage,
+    onDoubleClick,
+    fontSize,
+    aboveMessage,
+    belowMessage,
+    markdownProps,
+    actionsWrapWidth = 54,
+    ...rest
+  }) => {
+    const { mobile } = useResponsive();
+    const { cx, styles } = useStyles({
+      editing,
+      placement,
+      primary,
+      showTitle,
+      time,
+      title: avatar.title,
+      variant,
+    });
+
+    // 在 ChatItem 组件中添加
+    const contentRef = useRef<HTMLDivElement>(null);
+    const containerRef = useRef<HTMLDivElement>(null);
+    const [layoutMode, setLayoutMode] = useState<'horizontal' | 'vertical'>(
+      variant === 'bubble' ? 'horizontal' : 'vertical',
+    );
+
+    // 使用 ResizeObserver 监控内容和容器尺寸
+    useEffect(() => {
+      if (variant === 'docs') {
+        setLayoutMode('vertical');
+        return;
+      }
+
+      if (!contentRef.current || !containerRef.current) return;
+
+      const observer = new ResizeObserver(() => {
+        if (!contentRef.current || !containerRef.current) return;
+
+        const containerWidth = containerRef.current.clientWidth;
+        const contentWidth = contentRef.current.scrollWidth; // 使用scrollWidth获取实际内容宽度
+
+        // 预留给Actions的最小空间 (根据实际Actions大小调整)
+
+        // 只有当内容宽度 + Actions最小宽度 > 容器宽度时才切换布局
+        setLayoutMode(contentWidth + actionsWrapWidth > containerWidth ? 'vertical' : 'horizontal');
+      });
+
+      observer.observe(contentRef.current);
+      observer.observe(containerRef.current);
+
+      return () => observer.disconnect();
+    }, [variant, actionsWrapWidth]);
+
+    return (
+      <Flexbox
+        className={cx(styles.container, className)}
+        direction={placement === 'left' ? 'horizontal' : 'horizontal-reverse'}
+        gap={mobile ? 6 : 12}
+        {...rest}
+      >
+        <Avatar
+          {...avatarProps}
+          addon={avatarAddon}
+          alt={avatarProps?.alt || avatar.title || 'avatar'}
+          avatar={avatar}
+          loading={loading}
+          onClick={onAvatarClick}
+          placement={placement}
+          size={mobile ? MOBILE_AVATAR_SIZE : undefined}
+          style={{
+            marginTop: 6,
+            ...avatarProps?.style,
+          }}
+        />
+        <Flexbox
+          align={placement === 'left' ? 'flex-start' : 'flex-end'}
+          className={styles.messageContainer}
+          ref={containerRef}
+        >
+          <Title avatar={avatar} placement={placement} showTitle={showTitle} time={time} />
+          {aboveMessage}
+          <Flexbox
+            align={placement === 'left' ? 'flex-start' : 'flex-end'}
+            className={styles.messageContent}
+            data-layout={layoutMode} // 添加数据属性以方便样式选择
+            direction={
+              layoutMode === 'horizontal'
+                ? placement === 'left'
+                  ? 'horizontal'
+                  : 'horizontal-reverse'
+                : 'vertical'
+            }
+            gap={8}
+          >
+            <Flexbox ref={contentRef} width={'100%'}>
+              {error && (message === placeholderMessage || !message) ? (
+                <ErrorContent error={error} message={errorMessage} placement={placement} />
+              ) : (
+                <MessageContent
+                  editing={editing}
+                  fontSize={fontSize}
+                  markdownProps={markdownProps}
+                  message={message}
+                  messageExtra={
+                    <>
+                      {error && (
+                        <ErrorContent error={error} message={errorMessage} placement={placement} />
+                      )}
+                      {messageExtra}
+                    </>
+                  }
+                  onChange={onChange}
+                  onDoubleClick={onDoubleClick}
+                  onEditingChange={onEditingChange}
+                  placement={placement}
+                  primary={primary}
+                  renderMessage={renderMessage}
+                  text={text}
+                  variant={variant}
+                />
+              )}
+            </Flexbox>
+            {actions && (
+              <Actions
+                actions={actions}
+                editing={editing}
+                placement={placement}
+                variant={variant}
+              />
+            )}
+          </Flexbox>
+          {belowMessage}
+        </Flexbox>
+        {mobile && variant === 'bubble' && <BorderSpacing borderSpacing={MOBILE_AVATAR_SIZE} />}
+      </Flexbox>
+    );
+  },
+);
+
+export default ChatItem;
+
+export type { ChatItemProps } from './type';

package/src/components/ChatItem/components/Actions.tsx

@@ -0,0 +1,25 @@
+import { type Ref, memo } from 'react';
+import { Flexbox } from 'react-layout-kit';
+
+import { useStyles } from '../style';
+import { ChatItemProps } from '../type';
+
+export interface ActionsProps {
+  actions: ChatItemProps['actions'];
+  editing?: boolean;
+  placement?: ChatItemProps['placement'];
+  ref?: Ref<HTMLDivElement>;
+  variant?: ChatItemProps['variant'];
+}
+
+const Actions = memo<ActionsProps>(({ actions, placement, variant, editing, ref }) => {
+  const { styles } = useStyles({ editing, placement, variant });
+
+  return (
+    <Flexbox align={'flex-start'} className={styles.actions} ref={ref} role="menubar">
+      {actions}
+    </Flexbox>
+  );
+});
+
+export default Actions;

package/src/components/ChatItem/components/Avatar.tsx

@@ -0,0 +1,50 @@
+import { Avatar as A } from '@lobehub/ui';
+import { type CSSProperties, memo } from 'react';
+import { Flexbox } from 'react-layout-kit';
+
+import { useStyles } from '../style';
+import type { ChatItemProps } from '../type';
+import Loading from './Loading';
+
+export interface AvatarProps {
+  addon?: ChatItemProps['avatarAddon'];
+  alt?: string;
+  avatar: ChatItemProps['avatar'];
+  loading?: ChatItemProps['loading'];
+  onClick?: ChatItemProps['onAvatarClick'];
+  placement?: ChatItemProps['placement'];
+  size?: number;
+  style?: CSSProperties;
+  unoptimized?: boolean;
+}
+
+const Avatar = memo<AvatarProps>(
+  ({ loading, avatar, placement, unoptimized, addon, onClick, size = 40, style, alt }) => {
+    const { styles } = useStyles({ avatarSize: size });
+    const avatarContent = (
+      <div className={styles.avatarContainer} style={style}>
+        <A
+          alt={alt || avatar.title}
+          animation={loading}
+          avatar={avatar.avatar}
+          background={avatar.backgroundColor}
+          onClick={onClick}
+          size={size}
+          title={avatar.title}
+          unoptimized={unoptimized}
+        />
+        <Loading loading={loading} placement={placement} />
+      </div>
+    );
+
+    if (!addon) return avatarContent;
+    return (
+      <Flexbox align={'center'} className={styles.avatarGroupContainer} gap={8}>
+        {avatarContent}
+        {addon}
+      </Flexbox>
+    );
+  },
+);
+
+export default Avatar;

package/src/components/ChatItem/components/BorderSpacing.tsx

@@ -0,0 +1,13 @@
+import { memo } from 'react';
+
+export interface BorderSpacingProps {
+  borderSpacing?: number;
+}
+
+const BorderSpacing = memo<BorderSpacingProps>(({ borderSpacing }) => {
+  if (!borderSpacing) return null;
+
+  return <div style={{ flex: 'none', width: borderSpacing }} />;
+});
+
+export default BorderSpacing;

package/src/components/ChatItem/components/ErrorContent.tsx

@@ -0,0 +1,24 @@
+import { Alert } from '@lobehub/ui';
+import { memo } from 'react';
+import { Flexbox } from 'react-layout-kit';
+
+import { useStyles } from '../style';
+import { ChatItemProps } from '../type';
+
+export interface ErrorContentProps {
+  error?: ChatItemProps['error'];
+  message?: ChatItemProps['errorMessage'];
+  placement?: ChatItemProps['placement'];
+}
+
+const ErrorContent = memo<ErrorContentProps>(({ message, error, placement }) => {
+  const { styles } = useStyles({ placement });
+
+  return (
+    <Flexbox className={styles.errorContainer}>
+      <Alert closable={false} extra={message} showIcon type={'error'} {...error} />
+    </Flexbox>
+  );
+});
+
+export default ErrorContent;

package/src/components/ChatItem/components/Loading.tsx

@@ -0,0 +1,26 @@
+import { Icon } from '@lobehub/ui';
+import { Loader2 } from 'lucide-react';
+import { memo } from 'react';
+import { Flexbox } from 'react-layout-kit';
+
+import { useStyles } from '../style';
+import { ChatItemProps } from '../type';
+
+export interface LoadingProps {
+  loading?: ChatItemProps['loading'];
+  placement?: ChatItemProps['placement'];
+}
+
+const Loading = memo<LoadingProps>(({ loading, placement }) => {
+  const { styles } = useStyles({ placement });
+
+  if (!loading) return null;
+
+  return (
+    <Flexbox align={'center'} className={styles.loading} justify={'center'}>
+      <Icon icon={Loader2} size={{ size: 12, strokeWidth: 3 }} spin />
+    </Flexbox>
+  );
+});
+
+export default Loading;

package/src/components/ChatItem/components/MessageContent.tsx

@@ -0,0 +1,76 @@
+import { MarkdownProps } from '@lobehub/ui';
+import { EditableMessage } from '@lobehub/ui/chat';
+import { useResponsive } from 'antd-style';
+import { type ReactNode, memo } from 'react';
+import { Flexbox } from 'react-layout-kit';
+
+import { useStyles } from '../style';
+import { ChatItemProps } from '../type';
+
+export interface MessageContentProps {
+  editing?: ChatItemProps['editing'];
+  fontSize?: number;
+  markdownProps?: Omit<MarkdownProps, 'className' | 'style' | 'children'>;
+  message?: ReactNode;
+  messageExtra?: ChatItemProps['messageExtra'];
+  onChange?: ChatItemProps['onChange'];
+  onDoubleClick?: ChatItemProps['onDoubleClick'];
+  onEditingChange?: ChatItemProps['onEditingChange'];
+  placement?: ChatItemProps['placement'];
+  primary?: ChatItemProps['primary'];
+  renderMessage?: ChatItemProps['renderMessage'];
+  text?: ChatItemProps['text'];
+  variant?: ChatItemProps['variant'];
+}
+
+const MessageContent = memo<MessageContentProps>(
+  ({
+    editing,
+    onChange,
+    onEditingChange,
+    text,
+    message,
+    placement,
+    messageExtra,
+    renderMessage,
+    variant,
+    primary,
+    onDoubleClick,
+    fontSize,
+    markdownProps,
+  }) => {
+    const { cx, styles } = useStyles({ editing, placement, primary, variant });
+    const { mobile } = useResponsive();
+
+    const content = (
+      <EditableMessage
+        classNames={{ input: styles.editingInput }}
+        editButtonSize={'small'}
+        editing={editing}
+        fontSize={fontSize}
+        fullFeaturedCodeBlock
+        markdownProps={markdownProps}
+        onChange={onChange}
+        onEditingChange={onEditingChange}
+        openModal={mobile ? editing : undefined}
+        text={text}
+        value={message ? String(message) : ''}
+      />
+    );
+    const messageContent = renderMessage ? renderMessage(content) : content;
+
+    return (
+      <Flexbox
+        className={cx(styles.message, editing && styles.editingContainer)}
+        onDoubleClick={onDoubleClick}
+      >
+        {messageContent}
+        {messageExtra && !editing ? (
+          <div className={styles.messageExtra}>{messageExtra}</div>
+        ) : null}
+      </Flexbox>
+    );
+  },
+);
+
+export default MessageContent;