@lobehub/chat 1.128.0 → 1.128.1

package/packages/context-engine/src/processors/MessageContent.ts

@@ -0,0 +1,298 @@
+import { filesPrompts } from '@lobechat/prompts';
+import { imageUrlToBase64, isLocalUrl, parseDataUri } from '@lobechat/utils';
+import debug from 'debug';
+
+import { BaseProcessor } from '../base/BaseProcessor';
+import type { PipelineContext, ProcessorOptions } from '../types';
+
+const log = debug('context-engine:processor:MessageContentProcessor');
+
+export interface FileContextConfig {
+  /** Whether to enable file context injection */
+  enabled?: boolean;
+  /** Whether to include file URLs in file context prompts */
+  includeFileUrl?: boolean;
+}
+
+export interface MessageContentConfig {
+  /** File context configuration */
+  fileContext?: FileContextConfig;
+  /** Function to check if vision is supported */
+  isCanUseVision?: (model: string, provider: string) => boolean | undefined;
+  /** Model name */
+  model: string;
+  /** Provider name */
+  provider: string;
+}
+
+export interface UserMessageContentPart {
+  image_url?: {
+    detail?: string;
+    url: string;
+  };
+  signature?: string;
+  text?: string;
+  thinking?: string;
+  type: 'text' | 'image_url' | 'thinking';
+}
+
+/**
+ * Message Content Processor
+ * Responsible for handling content format conversion of user and assistant messages
+ */
+export class MessageContentProcessor extends BaseProcessor {
+  readonly name = 'MessageContentProcessor';
+
+  constructor(
+    private config: MessageContentConfig,
+    options: ProcessorOptions = {},
+  ) {
+    super(options);
+  }
+
+  protected async doProcess(context: PipelineContext): Promise<PipelineContext> {
+    const clonedContext = this.cloneContext(context);
+
+    let processedCount = 0;
+    let userMessagesProcessed = 0;
+    let assistantMessagesProcessed = 0;
+
+    // 处理每条消息的内容
+    for (let i = 0; i < clonedContext.messages.length; i++) {
+      const message = clonedContext.messages[i];
+
+      try {
+        let updatedMessage = message;
+
+        if (message.role === 'user') {
+          updatedMessage = await this.processUserMessage(message);
+          if (updatedMessage !== message) {
+            userMessagesProcessed++;
+            processedCount++;
+          }
+        } else if (message.role === 'assistant') {
+          updatedMessage = await this.processAssistantMessage(message);
+          if (updatedMessage !== message) {
+            assistantMessagesProcessed++;
+            processedCount++;
+          }
+        }
+
+        if (updatedMessage !== message) {
+          clonedContext.messages[i] = updatedMessage;
+          log(`Processed message content ${message.id}, role: ${message.role}`);
+        }
+      } catch (error) {
+        log.extend('error')(`Error processing message ${message.id} content: ${error}`);
+        // 继续处理其他消息
+      }
+    }
+
+    // 更新元数据
+    clonedContext.metadata.messageContentProcessed = processedCount;
+    clonedContext.metadata.userMessagesProcessed = userMessagesProcessed;
+    clonedContext.metadata.assistantMessagesProcessed = assistantMessagesProcessed;
+
+    log(
+      `Message content processing completed, processed ${processedCount} messages (user: ${userMessagesProcessed}, assistant: ${assistantMessagesProcessed})`,
+    );
+
+    return this.markAsExecuted(clonedContext);
+  }
+
+  /**
+   * Process user message content
+   */
+  private async processUserMessage(message: any): Promise<any> {
+    // Check if images or files need processing
+    const hasImages = message.imageList && message.imageList.length > 0;
+    const hasFiles = message.fileList && message.fileList.length > 0;
+
+    // If no images and files, return plain text content directly
+    if (!hasImages && !hasFiles) {
+      return {
+        ...message,
+        content: message.content,
+      };
+    }
+
+    const contentParts: UserMessageContentPart[] = [];
+
+    // Add text content
+    let textContent = message.content || '';
+
+    // Add file context (if file context is enabled and has files or images)
+    if ((hasFiles || hasImages) && this.config.fileContext?.enabled) {
+      const filesContext = filesPrompts({
+        addUrl: this.config.fileContext.includeFileUrl ?? true,
+        fileList: message.fileList,
+        imageList: message.imageList,
+      });
+
+      if (filesContext) {
+        textContent = (textContent + '\n\n' + filesContext).trim();
+      }
+    }
+
+    // Add text part
+    if (textContent) {
+      contentParts.push({
+        text: textContent,
+        type: 'text',
+      });
+    }
+
+    // Process image content
+    if (hasImages && this.config.isCanUseVision?.(this.config.model, this.config.provider)) {
+      const imageContentParts = await this.processImageList(message.imageList || []);
+      contentParts.push(...imageContentParts);
+    }
+
+    // 明确返回的字段，只保留必要的消息字段
+    const hasFileContext = (hasFiles || hasImages) && this.config.fileContext?.enabled;
+    const hasVisionContent =
+      hasImages && this.config.isCanUseVision?.(this.config.model, this.config.provider);
+
+    // 如果只有文本内容且没有添加文件上下文也没有视觉内容，返回纯文本
+    if (
+      contentParts.length === 1 &&
+      contentParts[0].type === 'text' &&
+      !hasFileContext &&
+      !hasVisionContent
+    ) {
+      return {
+        content: contentParts[0].text,
+        createdAt: message.createdAt,
+        id: message.id,
+        meta: message.meta,
+        role: message.role,
+        updatedAt: message.updatedAt,
+        // 保留其他可能需要的字段，但移除已处理的文件相关字段
+        ...(message.tools && { tools: message.tools }),
+        ...(message.tool_calls && { tool_calls: message.tool_calls }),
+        ...(message.tool_call_id && { tool_call_id: message.tool_call_id }),
+        ...(message.name && { name: message.name }),
+      };
+    }
+
+    // 返回结构化内容
+    return {
+      content: contentParts,
+      createdAt: message.createdAt,
+      id: message.id,
+      meta: message.meta,
+      role: message.role,
+      updatedAt: message.updatedAt,
+      // 保留其他可能需要的字段，但移除已处理的文件相关字段
+      ...(message.tools && { tools: message.tools }),
+      ...(message.tool_calls && { tool_calls: message.tool_calls }),
+      ...(message.tool_call_id && { tool_call_id: message.tool_call_id }),
+      ...(message.name && { name: message.name }),
+    };
+  }
+
+  /**
+   * 处理助手消息内容
+   */
+  private async processAssistantMessage(message: any): Promise<any> {
+    // 检查是否有推理内容（thinking mode）
+    const shouldIncludeThinking = message.reasoning && !!message.reasoning?.signature;
+
+    if (shouldIncludeThinking) {
+      const contentParts: UserMessageContentPart[] = [
+        {
+          signature: message.reasoning!.signature,
+          thinking: message.reasoning!.content,
+          type: 'thinking',
+        },
+        {
+          text: message.content,
+          type: 'text',
+        },
+      ];
+
+      return {
+        ...message,
+        content: contentParts,
+      };
+    }
+
+    // 检查是否有图片（助手消息也可能包含图片）
+    const hasImages = message.imageList && message.imageList.length > 0;
+
+    if (hasImages && this.config.isCanUseVision?.(this.config.model, this.config.provider)) {
+      // 创建结构化内容
+      const contentParts: UserMessageContentPart[] = [];
+
+      if (message.content) {
+        contentParts.push({
+          text: message.content,
+          type: 'text',
+        });
+      }
+
+      // 处理图片内容
+      const imageContentParts = await this.processImageList(message.imageList || []);
+      contentParts.push(...imageContentParts);
+
+      return {
+        ...message,
+        content: contentParts,
+      };
+    }
+
+    // 普通助手消息，返回纯文本内容
+    return {
+      ...message,
+      content: message.content,
+    };
+  }
+
+  /**
+   * 处理图片列表
+   */
+  private async processImageList(imageList: any[]): Promise<UserMessageContentPart[]> {
+    if (!imageList || imageList.length === 0) {
+      return [];
+    }
+
+    return Promise.all(
+      imageList.map(async (image) => {
+        const { type } = parseDataUri(image.url);
+
+        let processedUrl = image.url;
+        if (type === 'url' && isLocalUrl(image.url)) {
+          const { base64, mimeType } = await imageUrlToBase64(image.url);
+          processedUrl = `data:${mimeType};base64,${base64}`;
+        }
+
+        return {
+          image_url: { detail: 'auto', url: processedUrl },
+          type: 'image_url',
+        } as UserMessageContentPart;
+      }),
+    );
+  }
+
+  /**
+   * 验证内容部分格式
+   */
+  private validateContentPart(part: UserMessageContentPart): boolean {
+    if (!part || !part.type) return false;
+
+    switch (part.type) {
+      case 'text': {
+        return typeof part.text === 'string';
+      }
+      case 'image_url': {
+        return !!(part.image_url && part.image_url.url);
+      }
+      case 'thinking': {
+        return !!(part.thinking && part.signature);
+      }
+      default: {
+        return false;
+      }
+    }
+  }
+}

package/packages/context-engine/src/processors/PlaceholderVariables.ts

@@ -0,0 +1,196 @@
+import debug from 'debug';
+
+import { BaseProcessor } from '../base/BaseProcessor';
+import type { PipelineContext, ProcessorOptions } from '../types';
+
+const log = debug('context-engine:processor:PlaceholderVariablesProcessor');
+
+const placeholderVariablesRegex = /{{(.*?)}}/g;
+
+export interface PlaceholderVariablesConfig {
+  /** Recursive parsing depth, default is 2 */
+  depth?: number;
+  /** Variable generators mapping, key is variable name, value is generator function */
+  variableGenerators: Record<string, () => string>;
+}
+
+/**
+ * Extract all {{variable}} placeholder variable names from text
+ * @param text String containing template variables
+ * @returns Array of variable names, e.g. ['date', 'nickname']
+ */
+const extractPlaceholderVariables = (text: string): string[] => {
+  const matches = [...text.matchAll(placeholderVariablesRegex)];
+  return matches.map((m) => m[1].trim());
+};
+
+/**
+ * Replace template variables with actual values, supporting recursive parsing of nested variables
+ * @param text - Original text containing variables
+ * @param variableGenerators - Variable generators mapping
+ * @param depth - Recursive depth, default 2, set higher to support {{date}} within {{text}}
+ * @returns Text with variables replaced
+ */
+export const parsePlaceholderVariables = (
+  text: string,
+  variableGenerators: Record<string, () => string>,
+  depth = 2,
+): string => {
+  let result = text;
+
+  // Recursive parsing to handle cases like {{text}} containing additional preset variables
+  for (let i = 0; i < depth; i++) {
+    try {
+      const extractedVariables = extractPlaceholderVariables(result);
+      const availableVariables = Object.fromEntries(
+        extractedVariables
+          .map((key) => [key, variableGenerators[key]?.()])
+          .filter(([, value]) => value !== undefined),
+      );
+
+      // Only perform replacement when there are available variables
+      if (Object.keys(availableVariables).length === 0) break;
+
+      // Replace variables one by one to avoid lodash template's error handling for undefined variables
+      let tempResult = result;
+      for (const [key, value] of Object.entries(availableVariables)) {
+        const regex = new RegExp(
+          `{{\\s*${key.replaceAll(/[$()*+.?[\\\]^{|}]/g, '\\$&')}\\s*}}`,
+          'g',
+        );
+        // @ts-ignore
+        tempResult = tempResult.replace(regex, value);
+      }
+
+      if (tempResult === result) break;
+      result = tempResult;
+    } catch {
+      break;
+    }
+  }
+
+  return result;
+};
+
+/**
+ * Parse message content and replace placeholder variables
+ * @param messages Original messages array
+ * @param variableGenerators Variable generators mapping
+ * @param depth Recursive parsing depth, default is 2
+ * @returns Processed messages array
+ */
+export const parsePlaceholderVariablesMessages = (
+  messages: any[],
+  variableGenerators: Record<string, () => string>,
+  depth = 2,
+): any[] =>
+  messages.map((message) => {
+    if (!message?.content) return message;
+
+    const { content } = message;
+
+    // Handle string type directly
+    if (typeof content === 'string') {
+      return { ...message, content: parsePlaceholderVariables(content, variableGenerators, depth) };
+    }
+
+    // Handle array type by processing text elements
+    if (Array.isArray(content)) {
+      return {
+        ...message,
+        content: content.map((item) =>
+          item?.type === 'text'
+            ? { ...item, text: parsePlaceholderVariables(item.text, variableGenerators, depth) }
+            : item,
+        ),
+      };
+    }
+
+    return message;
+  });
+
+/**
+ * PlaceholderVariables Processor
+ * Responsible for handling placeholder variable replacement in messages
+ */
+export class PlaceholderVariablesProcessor extends BaseProcessor {
+  readonly name = 'PlaceholderVariablesProcessor';
+
+  constructor(
+    private config: PlaceholderVariablesConfig,
+    options: ProcessorOptions = {},
+  ) {
+    super(options);
+  }
+
+  protected async doProcess(context: PipelineContext): Promise<PipelineContext> {
+    const clonedContext = this.cloneContext(context);
+
+    let processedCount = 0;
+    const depth = this.config.depth ?? 2;
+
+    log(
+      `Starting placeholder variables processing with ${Object.keys(this.config.variableGenerators).length} generators`,
+    );
+
+    // 处理每条消息的占位符变量
+    for (let i = 0; i < clonedContext.messages.length; i++) {
+      const message = clonedContext.messages[i];
+
+      try {
+        const originalMessage = JSON.stringify(message);
+        const processedMessage = this.processMessagePlaceholders(message, depth);
+
+        if (JSON.stringify(processedMessage) !== originalMessage) {
+          clonedContext.messages[i] = processedMessage;
+          processedCount++;
+          log(`Processed placeholders in message ${message.id}, role: ${message.role}`);
+        }
+      } catch (error) {
+        log.extend('error')(`Error processing placeholders in message ${message.id}: ${error}`);
+        // 继续处理其他消息
+      }
+    }
+
+    // 更新元数据
+    clonedContext.metadata.placeholderVariablesProcessed = processedCount;
+
+    log(`Placeholder variables processing completed, processed ${processedCount} messages`);
+
+    return this.markAsExecuted(clonedContext);
+  }
+
+  /**
+   * 处理单个消息的占位符变量
+   */
+  private processMessagePlaceholders(message: any, depth: number): any {
+    if (!message?.content) return message;
+
+    const { content } = message;
+
+    // Handle string type directly
+    if (typeof content === 'string') {
+      return {
+        ...message,
+        content: parsePlaceholderVariables(content, this.config.variableGenerators, depth),
+      };
+    }
+
+    // Handle array type by processing text elements
+    if (Array.isArray(content)) {
+      return {
+        ...message,
+        content: content.map((item) =>
+          item?.type === 'text'
+            ? {
+                ...item,
+                text: parsePlaceholderVariables(item.text, this.config.variableGenerators, depth),
+              }
+            : item,
+        ),
+      };
+    }
+
+    return message;
+  }
+}

package/packages/context-engine/src/processors/ToolCall.ts

@@ -0,0 +1,186 @@
+import debug from 'debug';
+
+import { BaseProcessor } from '../base/BaseProcessor';
+import type { MessageToolCall, PipelineContext, ProcessorOptions } from '../types';
+
+const log = debug('context-engine:processor:ToolCallProcessor');
+
+export interface ToolCallConfig {
+  /** Function to generate tool calling name */
+  genToolCallingName?: (identifier: string, apiName: string, type?: string) => string;
+  /** Function to check if function calling is supported */
+  isCanUseFC?: (model: string, provider: string) => boolean;
+  /** Model name */
+  model: string;
+  /** Provider name */
+  provider: string;
+}
+
+/**
+ * Tool Call Processor
+ * Responsible for converting ChatMessage format tool calls to OpenAI format
+ */
+export class ToolCallProcessor extends BaseProcessor {
+  readonly name = 'ToolCallProcessor';
+
+  constructor(
+    private config: ToolCallConfig,
+    options: ProcessorOptions = {},
+  ) {
+    super(options);
+  }
+
+  protected async doProcess(context: PipelineContext): Promise<PipelineContext> {
+    const clonedContext = this.cloneContext(context);
+
+    const supportTools = this.config.isCanUseFC
+      ? this.config.isCanUseFC(this.config.model, this.config.provider)
+      : true;
+
+    let processedCount = 0;
+    let toolCallsConverted = 0;
+    let toolMessagesConverted = 0;
+
+    // 处理每条消息的工具调用
+    for (let i = 0; i < clonedContext.messages.length; i++) {
+      const message = clonedContext.messages[i];
+
+      try {
+        const updatedMessage = await this.processMessage(message, supportTools);
+
+        if (updatedMessage !== message) {
+          processedCount++;
+          clonedContext.messages[i] = updatedMessage;
+
+          // 统计转换的工具调用和工具消息数量
+          if (message.role === 'assistant' && message.tools) {
+            toolCallsConverted += message.tools.length;
+          }
+          if (message.role === 'tool') {
+            toolMessagesConverted++;
+          }
+
+          log(`处理消息 ${message.id}，角色: ${message.role}`);
+        }
+      } catch (error) {
+        log.extend('error')(`处理消息 ${message.id} 工具调用时出错: ${error}`);
+        // 继续处理其他消息
+      }
+    }
+
+    // 更新元数据
+    clonedContext.metadata.toolCallProcessed = processedCount;
+    clonedContext.metadata.toolCallsConverted = toolCallsConverted;
+    clonedContext.metadata.toolMessagesConverted = toolMessagesConverted;
+    clonedContext.metadata.supportTools = supportTools;
+
+    log(
+      `Tool call processing completed, processed ${processedCount} messages, converted ${toolCallsConverted} tool calls, ${toolMessagesConverted} tool messages`,
+    );
+
+    return this.markAsExecuted(clonedContext);
+  }
+
+  /**
+   * 处理单条消息的工具调用
+   */
+  private async processMessage(message: any, supportTools: boolean): Promise<any> {
+    switch (message.role) {
+      case 'assistant': {
+        return this.processAssistantMessage(message, supportTools);
+      }
+
+      case 'tool': {
+        return this.processToolMessage(message, supportTools);
+      }
+
+      default: {
+        return message;
+      }
+    }
+  }
+
+  /**
+   * 处理助手消息的工具调用
+   */
+  private processAssistantMessage(message: any, supportTools: boolean): any {
+    // 检查是否有工具调用
+    const hasTools = message.tools && message.tools.length > 0;
+    const hasEmptyToolCalls = message.tool_calls && message.tool_calls.length === 0;
+
+    if (!supportTools || (!hasTools && hasEmptyToolCalls)) {
+      // 如果不支持工具或只有空的工具调用，返回普通消息（移除工具相关属性）
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      const { tools, tool_calls, ...messageWithoutTools } = message;
+      return messageWithoutTools;
+    }
+
+    if (!hasTools) {
+      // 如果没有 tools 但有其他工具调用属性，只移除 tools
+      return message;
+    }
+
+    // 将 tools 转换为 tool_calls 格式
+    const tool_calls = message.tools.map(
+      (tool: any): MessageToolCall => ({
+        function: {
+          arguments: tool.arguments,
+          name: this.config.genToolCallingName
+            ? this.config.genToolCallingName(tool.identifier, tool.apiName, tool.type)
+            : `${tool.identifier}.${tool.apiName}`,
+        },
+        id: tool.id,
+        type: 'function',
+      }),
+    );
+
+    return { ...message, tool_calls };
+  }
+
+  /**
+   * 处理工具消息
+   */
+  private processToolMessage(message: any, supportTools: boolean): any {
+    if (!supportTools) {
+      // 如果不支持工具，将工具消息转换为用户消息
+      return {
+        ...message,
+        name: undefined,
+        plugin: undefined,
+        role: 'user',
+        tool_call_id: undefined,
+      };
+    }
+
+    // 生成工具名称
+    const toolName = message.plugin
+      ? this.config.genToolCallingName
+        ? this.config.genToolCallingName(
+            message.plugin.identifier,
+            message.plugin.apiName,
+            message.plugin.type,
+          )
+        : `${message.plugin.identifier}.${message.plugin.apiName}`
+      : undefined;
+
+    return {
+      ...message,
+      name: toolName,
+      // 保留 tool_call_id 用于关联
+    };
+  }
+
+  /**
+   * 验证工具调用格式
+   */
+  private validateToolCall(tool: any): boolean {
+    return !!(tool && tool.id && tool.identifier && tool.apiName && tool.arguments);
+  }
+
+  /**
+   * 验证工具消息格式
+   */
+  private validateToolMessage(message: any): boolean {
+    return !!(message && message.tool_call_id && message.content !== undefined);
+  }
+}