@lobehub/lobehub 2.0.0-next.35 → 2.0.0-next.36

package/packages/database/src/utils/groupMessages.ts

@@ -1,361 +0,0 @@
-import {
-  AssistantContentBlock,
-  ChatToolPayloadWithResult,
-  MessageMetadata,
-  ModelPerformance,
-  ModelUsage,
-  UIChatMessage,
-} from '@lobechat/types';
-
-/**
- * Split MessageMetadata into usage and performance
- */
-function splitMetadata(metadata?: MessageMetadata | null): {
-  performance?: ModelPerformance;
-  usage?: ModelUsage;
-} {
-  if (!metadata) return {};
-
-  const usage: ModelUsage = {};
-  const performance: ModelPerformance = {};
-
-  // Extract usage fields (tokens and cost)
-  const usageFields = [
-    'inputCachedTokens',
-    'inputCacheMissTokens',
-    'inputWriteCacheTokens',
-    'inputTextTokens',
-    'inputImageTokens',
-    'inputAudioTokens',
-    'inputCitationTokens',
-    'outputTextTokens',
-    'outputImageTokens',
-    'outputAudioTokens',
-    'outputReasoningTokens',
-    'acceptedPredictionTokens',
-    'rejectedPredictionTokens',
-    'totalInputTokens',
-    'totalOutputTokens',
-    'totalTokens',
-    'cost',
-  ] as const;
-
-  let hasUsage = false;
-  usageFields.forEach((field) => {
-    if (metadata[field] !== undefined) {
-      usage[field] = metadata[field] as any;
-      hasUsage = true;
-    }
-  });
-
-  // Extract performance fields
-  const performanceFields = ['tps', 'ttft', 'duration', 'latency'] as const;
-  let hasPerformance = false;
-  performanceFields.forEach((field) => {
-    if (metadata[field] !== undefined) {
-      performance[field] = metadata[field];
-      hasPerformance = true;
-    }
-  });
-
-  return {
-    performance: hasPerformance ? performance : undefined,
-    usage: hasUsage ? usage : undefined,
-  };
-}
-
-/**
- * Aggregate metadata from all children blocks
- * Creates structured usage and performance metrics
- */
-function aggregateMetadata(
-  children: AssistantContentBlock[],
-): { performance?: ModelPerformance; usage?: ModelUsage } | null {
-  const usage: ModelUsage = {};
-  const performance: ModelPerformance = {};
-  let hasUsageData = false;
-  let hasPerformanceData = false;
-  let tpsSum = 0;
-  let tpsCount = 0;
-
-  children.forEach((child) => {
-    // Aggregate usage metrics (tokens and cost)
-    if (child.usage) {
-      const tokenFields = [
-        'inputCachedTokens',
-        'inputCacheMissTokens',
-        'inputWriteCacheTokens',
-        'inputTextTokens',
-        'inputImageTokens',
-        'inputAudioTokens',
-        'inputCitationTokens',
-        'outputTextTokens',
-        'outputImageTokens',
-        'outputAudioTokens',
-        'outputReasoningTokens',
-        'acceptedPredictionTokens',
-        'rejectedPredictionTokens',
-        'totalInputTokens',
-        'totalOutputTokens',
-        'totalTokens',
-      ] as const;
-
-      tokenFields.forEach((field) => {
-        if (typeof child.usage![field] === 'number') {
-          usage[field] = (usage[field] || 0) + child.usage![field]!;
-          hasUsageData = true;
-        }
-      });
-
-      if (typeof child.usage.cost === 'number') {
-        usage.cost = (usage.cost || 0) + child.usage.cost;
-        hasUsageData = true;
-      }
-    }
-
-    // Aggregate performance metrics
-    // - ttft: use the first child's value (time to first token)
-    // - tps: calculate average across all children
-    // - duration: sum all durations
-    // - latency: sum all latencies
-    if (child.performance) {
-      if (child.performance.ttft !== undefined && performance.ttft === undefined) {
-        performance.ttft = child.performance.ttft; // First child only
-        hasPerformanceData = true;
-      }
-      if (typeof child.performance.tps === 'number') {
-        tpsSum += child.performance.tps;
-        tpsCount += 1;
-        hasPerformanceData = true;
-      }
-      if (child.performance.duration !== undefined) {
-        performance.duration = (performance.duration || 0) + child.performance.duration;
-        hasPerformanceData = true;
-      }
-      if (child.performance.latency !== undefined) {
-        performance.latency = (performance.latency || 0) + child.performance.latency;
-        hasPerformanceData = true;
-      }
-    }
-  });
-
-  // Calculate average tps
-  if (tpsCount > 0) {
-    performance.tps = tpsSum / tpsCount;
-  }
-
-  // Return null if no data
-  if (!hasUsageData && !hasPerformanceData) return null;
-
-  // Return structured metrics
-  const result: { performance?: ModelPerformance; usage?: ModelUsage } = {};
-  if (hasUsageData) result.usage = usage;
-  if (hasPerformanceData) result.performance = performance;
-
-  return result;
-}
-
-/**
- * Group assistant messages with their tool results
- * Converts flat message list into grouped structure with children
- *
- * @param messages - Flat message list from database query
- * @returns Grouped message list with assistant children populated
- */
-export function groupAssistantMessages(messages: UIChatMessage[]): UIChatMessage[] {
-  const result: UIChatMessage[] = [];
-  const toolMessageIds = new Set<string>();
-  const processedAssistantIds = new Set<string>();
-
-  // 1. Create tool_call_id -> tool message mapping
-  const toolMessageMap = new Map<string, UIChatMessage>();
-  messages.forEach((msg) => {
-    if (msg.role === 'tool' && msg.tool_call_id) {
-      toolMessageMap.set(msg.tool_call_id, msg);
-    }
-  });
-
-  // 2. Build message ID -> message mapping for quick lookup
-  const messageMap = new Map<string, UIChatMessage>();
-  messages.forEach((msg) => {
-    messageMap.set(msg.id, msg);
-  });
-
-  // 3. Find follow-up assistants that have tool messages as parent
-  // Map: tool message id -> follow-up assistant messages
-  const toolToFollowUpAssistants = new Map<string, UIChatMessage[]>();
-  messages.forEach((msg) => {
-    if (msg.role === 'assistant' && msg.parentId) {
-      const parent = messageMap.get(msg.parentId);
-      if (parent && parent.role === 'tool') {
-        const existing = toolToFollowUpAssistants.get(msg.parentId) || [];
-        toolToFollowUpAssistants.set(msg.parentId, [...existing, msg]);
-      }
-    }
-  });
-
-  // 4. Process messages
-  messages.forEach((msg) => {
-    // Skip tool messages that have been merged
-    if (msg.role === 'tool') {
-      if (!toolMessageIds.has(msg.id)) {
-        result.push(msg);
-      }
-      return;
-    }
-
-    // Handle non-assistant messages
-    if (msg.role !== 'assistant') {
-      result.push(msg);
-      return;
-    }
-
-    // Skip assistant messages that have been processed as follow-ups
-    if (processedAssistantIds.has(msg.id)) {
-      return;
-    }
-
-    // 5. Process assistant messages
-    const assistantMsg = { ...msg };
-
-    // If no tools, add as-is (unless it's a follow-up, which should have been skipped above)
-    if (!msg.tools || msg.tools.length === 0) {
-      result.push(assistantMsg);
-      return;
-    }
-
-    // 6. Build children structure
-    const children: AssistantContentBlock[] = [];
-
-    // First child: original assistant with tool results
-    const toolsWithResults: ChatToolPayloadWithResult[] = msg.tools.map((tool) => {
-      const toolMsg = toolMessageMap.get(tool.id);
-
-      if (toolMsg) {
-        // Mark tool message as merged
-        toolMessageIds.add(toolMsg.id);
-
-        return {
-          ...tool,
-          result: {
-            content: toolMsg.content,
-            error: toolMsg.pluginError,
-            id: toolMsg.id,
-            state: toolMsg.pluginState,
-          },
-          result_msg_id: toolMsg.id,
-        };
-      }
-
-      // Tool message not yet available (still executing)
-      return tool;
-    });
-
-    const { usage: msgUsage, performance: msgPerformance } = splitMetadata(msg.metadata);
-    children.push({
-      content: msg.content || '',
-      error: msg.error,
-      id: msg.id,
-      imageList: msg.imageList && msg.imageList.length > 0 ? msg.imageList : undefined,
-      performance: msgPerformance,
-      reasoning: msg.reasoning || undefined,
-      tools: toolsWithResults,
-      usage: msgUsage,
-    });
-
-    // 7. Recursively add follow-up assistants as additional children
-    // Keep track of tool result IDs that are part of this group
-    const groupToolResultIds = new Set<string>();
-    toolsWithResults.forEach((tool) => {
-      if (tool.result) {
-        groupToolResultIds.add(tool.result.id);
-      }
-    });
-
-    // Recursively collect all follow-up assistants
-    const collectFollowUpAssistants = (currentToolResultIds: Set<string>): void => {
-      const newToolResultIds = new Set<string>();
-
-      currentToolResultIds.forEach((toolResultId) => {
-        const followUps = toolToFollowUpAssistants.get(toolResultId) || [];
-        followUps.forEach((followUpMsg) => {
-          // Skip if already processed
-          if (processedAssistantIds.has(followUpMsg.id)) return;
-
-          // Process follow-up assistant's tools and fill in their results
-          const followUpToolsWithResults: ChatToolPayloadWithResult[] | undefined =
-            followUpMsg.tools?.map((followUpTool) => {
-              const followUpToolMsg = toolMessageMap.get(followUpTool.id);
-
-              if (followUpToolMsg) {
-                // Mark tool message as merged
-                toolMessageIds.add(followUpToolMsg.id);
-                // Track this tool result for next iteration
-                newToolResultIds.add(followUpToolMsg.id);
-
-                return {
-                  ...followUpTool,
-                  result: {
-                    content: followUpToolMsg.content,
-                    error: followUpToolMsg.pluginError,
-                    id: followUpToolMsg.id,
-                    state: followUpToolMsg.pluginState,
-                  },
-                  result_msg_id: followUpToolMsg.id,
-                };
-              }
-
-              // Tool message not yet available
-              return followUpTool;
-            });
-
-          const { usage: followUpUsage, performance: followUpPerformance } = splitMetadata(
-            followUpMsg.metadata,
-          );
-          children.push({
-            content: followUpMsg.content || '',
-            error: followUpMsg.error,
-            id: followUpMsg.id,
-            imageList:
-              followUpMsg.imageList && followUpMsg.imageList.length > 0
-                ? followUpMsg.imageList
-                : undefined,
-            performance: followUpPerformance,
-            reasoning: followUpMsg.reasoning || undefined,
-            tools: followUpToolsWithResults,
-            usage: followUpUsage,
-          });
-          processedAssistantIds.add(followUpMsg.id);
-        });
-      });
-
-      // Recursively process the next level
-      if (newToolResultIds.size > 0) {
-        collectFollowUpAssistants(newToolResultIds);
-      }
-    };
-
-    collectFollowUpAssistants(groupToolResultIds);
-
-    // 8. Aggregate usage and performance from all children
-    const aggregated = aggregateMetadata(children);
-
-    // 9. Set children and aggregated metrics
-    assistantMsg.role = 'group';
-    assistantMsg.children = children;
-    if (aggregated) {
-      assistantMsg.usage = aggregated.usage;
-      assistantMsg.performance = aggregated.performance;
-    }
-    delete assistantMsg.metadata; // Clear individual metadata
-    delete assistantMsg.reasoning; // Reasoning moved to children blocks
-    delete assistantMsg.tools;
-    delete assistantMsg.imageList;
-    delete assistantMsg.fileList;
-    assistantMsg.content = ''; // Content moved to children
-
-    result.push(assistantMsg);
-  });
-
-  return result;
-}

package/packages/utils/src/tokenizer/client.ts

@@ -1,35 +0,0 @@
-let worker: Worker | null = null;
-
-const getWorker = () => {
-  if (!worker && typeof Worker !== 'undefined') {
-    worker = new Worker(new URL('tokenizer.worker.ts', import.meta.url));
-  }
-  return worker;
-};
-
-export const clientEncodeAsync = (str: string): Promise<number> =>
-  new Promise((resolve, reject) => {
-    const worker = getWorker();
-
-    if (!worker) {
-      // 如果 WebWorker 不可用，回退到字符串计算
-      resolve(str.length);
-      return;
-    }
-
-    const id = str;
-
-    const handleMessage = (event: MessageEvent) => {
-      if (event.data.id === id) {
-        worker.removeEventListener('message', handleMessage);
-        if (event.data.error) {
-          reject(new Error(event.data.error));
-        } else {
-          resolve(event.data.result);
-        }
-      }
-    };
-
-    worker.addEventListener('message', handleMessage);
-    worker.postMessage({ id, str });
-  });

package/packages/utils/src/tokenizer/estimated.ts

@@ -1,4 +0,0 @@
-import { approximateTokenSize } from 'tokenx';
-
-export const estimatedEncodeAsync = async (str: string): Promise<number> =>
-  approximateTokenSize(str);

package/packages/utils/src/tokenizer/server.ts

@@ -1,11 +0,0 @@
-export const serverEncodeAsync = async (str: string): Promise<number> => {
-  try {
-    const res = await fetch('/webapi/tokenizer', { body: str, method: 'POST' });
-    const data = await res.json();
-
-    return data.count;
-  } catch (e) {
-    console.error('serverEncodeAsync:', e);
-    return str.length;
-  }
-};

package/packages/utils/src/tokenizer/tokenizer.worker.ts

@@ -1,12 +0,0 @@
-addEventListener('message', async (event) => {
-  const { id, str } = event.data;
-  try {
-    const { encode } = await import('gpt-tokenizer');
-
-    const tokenCount = encode(str).length;
-
-    postMessage({ id, result: tokenCount });
-  } catch (error) {
-    postMessage({ error: (error as Error).message, id });
-  }
-});

package/src/app/(backend)/webapi/tokenizer/index.test.ts

@@ -1,32 +0,0 @@
-// @vitest-environment edge-runtime
-import { describe, expect, it } from 'vitest';
-
-import { POST } from './route';
-
-describe('tokenizer Route', () => {
-  it('count hello world', async () => {
-    const txt = 'Hello, world!';
-    const request = new Request('https://test.com', {
-      method: 'POST',
-      body: txt,
-    });
-
-    const response = await POST(request);
-
-    const data = await response.json();
-    expect(data.count).toEqual(4);
-  });
-
-  it('count Chinese', async () => {
-    const txt = '今天天气真好';
-    const request = new Request('https://test.com', {
-      method: 'POST',
-      body: txt,
-    });
-
-    const response = await POST(request);
-
-    const data = await response.json();
-    expect(data.count).toEqual(5);
-  });
-});

package/src/app/(backend)/webapi/tokenizer/route.ts

@@ -1,8 +0,0 @@
-import { encode } from 'gpt-tokenizer/encoding/o200k_base';
-import { NextResponse } from 'next/server';
-
-export const POST = async (req: Request) => {
-  const str = await req.text();
-
-  return NextResponse.json({ count: encode(str).length });
-};

package/src/features/Conversation/Error/InvalidAccessCode.tsx

@@ -1,79 +0,0 @@
-import { Icon, Segmented } from '@lobehub/ui';
-import { SegmentedLabeledOption } from 'antd/es/segmented';
-import { AsteriskSquare, KeySquare, ScanFace } from 'lucide-react';
-import { memo, useState } from 'react';
-import { useTranslation } from 'react-i18next';
-import { Flexbox } from 'react-layout-kit';
-
-import { useServerConfigStore } from '@/store/serverConfig';
-import { featureFlagsSelectors, serverConfigSelectors } from '@/store/serverConfig/selectors';
-
-import AccessCodeForm from './AccessCodeForm';
-import ChatInvalidAPIKey from './ChatInvalidApiKey';
-import OAuthForm from './OAuthForm';
-import { ErrorActionContainer } from './style';
-
-enum Tab {
-  Api = 'api',
-  Oauth = 'oauth',
-  Password = 'password',
-}
-
-interface InvalidAccessCodeProps {
-  id: string;
-  provider?: string;
-}
-
-const InvalidAccessCode = memo<InvalidAccessCodeProps>(({ id, provider }) => {
-  const { t } = useTranslation('error');
-  const isEnabledOAuth = useServerConfigStore(serverConfigSelectors.enabledOAuthSSO);
-  const defaultTab = isEnabledOAuth ? Tab.Oauth : Tab.Password;
-  const [mode, setMode] = useState<Tab>(defaultTab);
-  const { showOpenAIApiKey } = useServerConfigStore(featureFlagsSelectors);
-  const isEnabledTab = showOpenAIApiKey || isEnabledOAuth;
-
-  return (
-    <ErrorActionContainer>
-      {isEnabledTab && (
-        <Segmented
-          block
-          onChange={(value) => setMode(value as Tab)}
-          options={
-            [
-              isEnabledOAuth
-                ? {
-                    icon: <Icon icon={ScanFace} />,
-                    label: t('oauth', { ns: 'common' }),
-                    value: Tab.Oauth,
-                  }
-                : undefined,
-              {
-                icon: <Icon icon={AsteriskSquare} />,
-                label: t('unlock.tabs.password'),
-                value: Tab.Password,
-              },
-              showOpenAIApiKey
-                ? {
-                    icon: <Icon icon={KeySquare} />,
-                    label: t('unlock.tabs.apiKey'),
-                    value: Tab.Api,
-                  }
-                : undefined,
-            ].filter(Boolean) as SegmentedLabeledOption[]
-          }
-          style={{ width: '100%' }}
-          value={mode}
-          variant={'filled'}
-        />
-      )}
-
-      <Flexbox gap={24}>
-        {mode === Tab.Password && <AccessCodeForm id={id} />}
-        {showOpenAIApiKey && mode === Tab.Api && <ChatInvalidAPIKey id={id} provider={provider} />}
-        {isEnabledOAuth && mode === Tab.Oauth && <OAuthForm id={id} />}
-      </Flexbox>
-    </ErrorActionContainer>
-  );
-});
-
-export default InvalidAccessCode;