@lobehub/chat 1.134.3 → 1.134.4

package/packages/prompts/promptfoo/emoji-picker/eval.yaml

@@ -0,0 +1,170 @@
+description: Test emoji selection for different conversation topics and contexts
+
+providers:
+  - openai:chat:gpt-5-mini
+  - openai:chat:claude-3-5-haiku-latest
+  - openai:chat:gemini-flash-latest
+  - openai:chat:deepseek-chat
+
+prompts:
+  - file://promptfoo/emoji-picker/prompt.ts
+
+tests:
+  - vars:
+      content: "I just got a promotion at work! I'm so excited!"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should select a positive, celebratory emoji appropriate for work success (e.g., 🎉, 🎊, ⭐, 💼, 🚀)"
+      - type: not-contains
+        value: "explanation"
+
+  - vars:
+      content: "My dog passed away yesterday. I'm really sad."
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should select a sympathetic, sad emoji appropriate for loss and grief (e.g., 😢, 😭, 💔, 😔)"
+      - type: not-contains
+        value: "explanation"
+
+  - vars:
+      content: "Can you help me with this math problem?"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should select an emoji related to learning, thinking, or mathematics (e.g., 🤔, 📚, ✏️, 🧮, 🔢)"
+      - type: not-contains
+        value: "explanation"
+
+  - vars:
+      content: "I'm going on vacation to Hawaii next week!"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should select an emoji related to vacation, travel, or tropical themes (e.g., 🌺, 🏖️, ✈️, 🌴, ☀️)"
+      - type: not-contains
+        value: "explanation"
+
+  - vars:
+      content: "I'm learning to cook Italian food"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should select an emoji related to cooking or Italian food (e.g., 🍝, 🍕, 👨‍🍳, 🍽️)"
+      - type: not-contains
+        value: "explanation"
+
+  - vars:
+      content: "Technical documentation about API endpoints"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should select an emoji related to technology, development, or documentation (e.g., 💻, 📖, ⚙️, 🔧, 📝)"
+      - type: not-contains
+        value: "explanation"
+
+  # Chinese language tests
+  - vars:
+      content: "我刚刚升职了！太激动了！"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should select a positive, celebratory emoji appropriate for work success (e.g., 🎉, 🎊, ⭐, 💼, 🚀)"
+      - type: not-contains
+        value: "explanation"
+
+  - vars:
+      content: "我的猫咪昨天去世了，我很难过"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should select a sympathetic, sad emoji appropriate for loss and grief (e.g., 😢, 😭, 💔, 😔)"
+      - type: not-contains
+        value: "explanation"
+
+  - vars:
+      content: "我正在学习做日本料理"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should select an emoji related to cooking or Japanese food (e.g., 🍱, 🍣, 🍜, 👨‍🍳)"
+      - type: not-contains
+        value: "explanation"
+
+  # Spanish language tests
+  - vars:
+      content: "¡Me voy de vacaciones a la playa la próxima semana!"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should select an emoji related to vacation, beach, or tropical themes (e.g., 🏖️, ☀️, 🌊, 🏝️)"
+      - type: not-contains
+        value: "explanation"
+
+  - vars:
+      content: "Estoy estudiando para mi examen de matemáticas"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should select an emoji related to studying, learning, or mathematics (e.g., 📚, 🤓, 🧮, ✏️, 📊, 📐, 📏)"
+      - type: not-contains
+        value: "explanation"
+
+  # French language tests
+  - vars:
+      content: "Je viens de terminer mon marathon! Je suis épuisé mais heureux"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should select an emoji related to running, sports, or achievement (e.g., 🏃, 🏅, 💪, 🎯)"
+      - type: not-contains
+        value: "explanation"
+
+  - vars:
+      content: "J'apprends à jouer de la guitare"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should select an emoji related to music or guitar (e.g., 🎸, 🎵, 🎶, 🎼)"
+      - type: not-contains
+        value: "explanation"
+
+  # Japanese language tests
+  - vars:
+      content: "新しいプロジェクトが始まりました！頑張ります"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should select an emoji related to new beginning, work, or motivation (e.g., 🚀, 💼, 💪, ✨)"
+      - type: not-contains
+        value: "explanation"
+
+  - vars:
+      content: "桜が咲いて本当に綺麗です"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should select an emoji related to cherry blossoms, flowers, or beauty (e.g., 🌸, 🌺, 🌼, 🌷)"
+      - type: not-contains
+        value: "explanation"
+
+  # German language tests
+  - vars:
+      content: "Ich habe gerade ein neues Auto gekauft!"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should select an emoji related to cars or excitement (e.g., 🚗, 🎉, 🚙)"
+      - type: not-contains
+        value: "explanation"
+
+  # Russian language tests
+  - vars:
+      content: "Я люблю читать книги по вечерам"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should select an emoji related to reading or books (e.g., 📚, 📖, 📕, 🤓)"
+      - type: not-contains
+        value: "explanation"

package/packages/prompts/promptfoo/emoji-picker/prompt.ts

@@ -0,0 +1,16 @@
+// TypeScript prompt wrapper that uses actual chain implementation
+import { chainPickEmoji } from '@lobechat/prompts';
+
+interface PromptVars {
+  content: string;
+}
+
+export default function generatePrompt({ vars }: { vars: PromptVars }) {
+  const { content } = vars;
+
+  // Use the actual chain function from src
+  const result = chainPickEmoji(content);
+
+  // Return messages array as expected by promptfoo
+  return result.messages || [];
+}

package/packages/prompts/promptfoo/knowledge-qa/eval.yaml

@@ -0,0 +1,89 @@
+description: Test knowledge base Q&A with context retrieval and answer generation
+
+providers:
+  - openai:chat:gpt-5-mini
+  - openai:chat:claude-3-5-haiku-latest
+  - openai:chat:gemini-flash-latest
+  - openai:chat:deepseek-chat
+
+prompts:
+  - file://promptfoo/knowledge-qa/prompt.ts
+
+tests:
+  - vars:
+      context: "React is a JavaScript library for building user interfaces. It was developed by Facebook and is now maintained by Facebook and the community. React uses a virtual DOM to efficiently update and render components. Key features include component-based architecture, JSX syntax, and state management through hooks."
+      query: "What is React and who developed it?"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "The response should accurately explain what React is and mention Facebook as the developer"
+      - type: contains-any
+        value: ["React", "JavaScript", "library", "Facebook"]
+      - type: not-contains
+        value: "I don't know"
+
+  - vars:
+      context: "TypeScript is a strongly typed programming language that builds on JavaScript by adding static type definitions. It was developed by Microsoft. TypeScript code compiles to clean, simple JavaScript code which runs on any browser, Node.js environment, or any JavaScript engine that supports ECMAScript 3 or newer."
+      query: "How does TypeScript relate to JavaScript?"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "The response should explain the relationship between TypeScript and JavaScript, mentioning type definitions"
+      - type: contains-any
+        value: ["TypeScript", "JavaScript", "type", "compiles", "strongly typed"]
+      - type: not-contains
+        value: "不知道"
+
+  - vars:
+      context: "Node.js是一个基于Chrome V8引擎的JavaScript运行时环境。它使用了事件驱动、非阻塞I/O模型，使其轻量而高效。Node.js的包管理器npm是世界上最大的开源库生态系统。"
+      query: "Node.js有什么特点？"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "The response should describe Node.js features in Chinese, mentioning event-driven and non-blocking I/O"
+      - type: contains-any
+        value: ["Node.js", "事件驱动", "非阻塞", "JavaScript", "运行时"]
+      - type: not-contains
+        value: "I don't know"
+
+  - vars:
+      context: "Docker is a containerization platform that allows developers to package applications and their dependencies into lightweight, portable containers. Containers share the OS kernel but run in isolated user spaces. This makes applications more consistent across different environments."
+      query: "How can I deploy my app with Docker?"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "The response should provide helpful information about deploying with Docker based on the containerization context. It should mention containers, packaging, or provide deployment-related guidance."
+      - type: contains-any
+        value: ["Docker", "container", "package", "deploy"]
+
+  - vars:
+      context: "GraphQL is a query language for APIs and a runtime for fulfilling those queries with existing data. Unlike REST APIs that require multiple requests to different endpoints, GraphQL allows clients to request exactly the data they need in a single request."
+      query: "What are the benefits of using GraphQL over REST?"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "The response should compare GraphQL and REST, highlighting GraphQL's advantages such as single request capability or requesting specific data"
+      - type: contains-any
+        value: ["GraphQL", "REST"]
+
+  - vars:
+      context: "Machine learning algorithms can be categorized into supervised, unsupervised, and reinforcement learning. Supervised learning uses labeled data to train models, unsupervised learning finds patterns in unlabeled data, and reinforcement learning learns through trial and error with rewards."
+      query: "Can you explain blockchain technology?"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "The response should indicate that the provided context is about machine learning, not blockchain, and cannot answer the blockchain question based on the context"
+      - type: contains-any
+        value: ["machine learning", "cannot", "不能", "no information", "context", "does not contain"]
+
+  - vars:
+      context: ""
+      query: "How do I set up a web server?"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "The response should provide helpful information about setting up a web server using general knowledge"
+      - type: contains-any
+        value: ["server", "web", "setup", "install", "configure"]
+      - type: not-contains
+        value: "cannot answer"

package/packages/prompts/promptfoo/knowledge-qa/prompt.ts

@@ -0,0 +1,26 @@
+// TypeScript prompt wrapper that uses actual chain implementation
+import { chainAnswerWithContext } from '@lobechat/prompts';
+
+interface PromptVars {
+  context: string | string[];
+  knowledge?: string | string[];
+  query: string;
+}
+
+export default function generatePrompt({ vars }: { vars: PromptVars }) {
+  const { context, query, knowledge = ['general knowledge'] } = vars;
+
+  // Ensure context and knowledge are arrays
+  const contextArray = Array.isArray(context) ? context : [context];
+  const knowledgeArray = Array.isArray(knowledge) ? knowledge : [knowledge];
+
+  // Use the actual chain function from src
+  const result = chainAnswerWithContext({
+    context: contextArray,
+    knowledge: knowledgeArray,
+    question: query,
+  });
+
+  // Return messages array as expected by promptfoo
+  return result.messages || [];
+}

package/packages/prompts/promptfoo/language-detection/eval.yaml

@@ -0,0 +1,65 @@
+description: Test language detection accuracy for various text inputs
+
+providers:
+  - openai:chat:gpt-5-mini
+  - openai:chat:claude-3-5-haiku-latest
+  - openai:chat:gemini-flash-latest
+  - openai:chat:deepseek-chat
+
+prompts:
+  - file://promptfoo/language-detection/prompt.ts
+
+tests:
+  - vars:
+      content: "Hello, how are you today? I hope you're having a great day!"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should output a valid locale code for English (e.g., en-US, en-GB, en)"
+      - type: contains-any
+        value: ["en-", "en"]
+
+  - vars:
+      content: "Bonjour, comment allez-vous? J'espère que vous passez une excellente journée!"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should output a valid locale code for French (e.g., fr-FR, fr-CA, fr)"
+      - type: contains-any
+        value: ["fr-", "fr"]
+
+  - vars:
+      content: "你好，你今天怎么样？希望你过得愉快！"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should output a valid locale code for Chinese (e.g., zh-CN, zh-TW, zh)"
+      - type: contains-any
+        value: ["zh-", "zh"]
+
+  - vars:
+      content: "Hola, ¿cómo estás hoy? ¡Espero que tengas un gran día!"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should output a valid locale code for Spanish (e.g., es-ES, es-MX, es)"
+      - type: contains-any
+        value: ["es-", "es"]
+
+  - vars:
+      content: "Привет, как дела сегодня? Надеюсь, у тебя отличный день!"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should output a valid locale code for Russian (e.g., ru-RU, ru)"
+      - type: contains-any
+        value: ["ru-", "ru"]
+
+  - vars:
+      content: "こんにちは、今日はいかがですか？素晴らしい一日をお過ごしください！"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "Should output a valid locale code for Japanese (e.g., ja-JP, ja)"
+      - type: contains-any
+        value: ["ja-", "ja"]

package/packages/prompts/promptfoo/language-detection/prompt.ts

@@ -0,0 +1,16 @@
+// TypeScript prompt wrapper that uses actual chain implementation
+import { chainLangDetect } from '@lobechat/prompts';
+
+interface PromptVars {
+  content: string;
+}
+
+export default function generatePrompt({ vars }: { vars: PromptVars }) {
+  const { content } = vars;
+
+  // Use the actual chain function from src
+  const result = chainLangDetect(content);
+
+  // Return messages array as expected by promptfoo
+  return result.messages || [];
+}

package/packages/prompts/promptfoo/summary-title/eval.yaml

@@ -0,0 +1,85 @@
+description: Test summary title generation for different conversation types
+
+providers:
+  - openai:chat:gpt-5-mini
+  - openai:chat:claude-3-5-haiku-latest
+  - openai:chat:gemini-flash-latest
+  - openai:chat:deepseek-chat
+
+prompts:
+  - file://promptfoo/summary-title/prompt.ts
+
+tests:
+  - vars:
+      messages:
+        - role: "user"
+          content: "How do I install Node.js on my computer?"
+        - role: "assistant"
+          content: "To install Node.js, you can download it from the official website nodejs.org and follow the installation instructions for your operating system."
+        - role: "user"
+          content: "What about using a version manager?"
+        - role: "assistant"
+          content: "Yes! I recommend using nvm (Node Version Manager) which allows you to install and switch between different Node.js versions easily."
+      locale: "en-US"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "The response should be a concise title (10 words or less) that summarizes the conversation about Node.js installation"
+      - type: regex
+        value: "^.{1,50}$"  # Title should be between 1-50 characters
+      - type: not-contains
+        value: "标点符号"  # Should not contain punctuation as requested in Chinese
+
+  - vars:
+      messages:
+        - role: "user"
+          content: "我想学习做蛋炒饭"
+        - role: "assistant"
+          content: "蛋炒饭是很经典的家常菜！你需要准备鸡蛋、米饭、葱花、盐和生抽等基本材料。"
+        - role: "user"
+          content: "具体步骤是什么？"
+        - role: "assistant"
+          content: "首先打散鸡蛋炒熟盛起，然后下米饭炒散，最后加入鸡蛋和调料炒匀即可。"
+      locale: "zh-CN"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+        value: "The response should be a Chinese title summarizing the conversation about fried rice cooking"
+      - type: regex
+        value: "^.{1,30}$"  # Chinese titles can be shorter
+      - type: contains-any
+        value: ["蛋炒饭", "做饭", "烹饪", "料理"]
+
+  - vars:
+      messages:
+        - role: "user"
+          content: "Can you help me debug this Python error?"
+        - role: "assistant"
+          content: "Of course! Please share the error message and the relevant code."
+        - role: "user"
+          content: "I'm getting 'AttributeError: 'NoneType' object has no attribute 'split''"
+        - role: "assistant"
+          content: "This error occurs when you're trying to call .split() on a None value. The variable is likely None instead of a string."
+      locale: "en-US"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+
+        value: "The response should be a title about Python debugging or error resolution"
+      - type: contains-any
+        value: ["Python", "debug", "error", "AttributeError", "code"]
+
+  - vars:
+      messages:
+        - role: "user"
+          content: "¿Cómo está el tiempo hoy?"
+        - role: "assistant"
+          content: "No tengo acceso a información meteorológica en tiempo real, pero puedes consultar el clima en tu área usando aplicaciones como Weather.com o tu app del tiempo local."
+      locale: "es-ES"
+    assert:
+      - type: llm-rubric
+        provider: openai:gpt-5-mini
+
+        value: "The response should be a Spanish title about weather inquiry"
+      - type: regex
+        value: "^.{1,50}$"

package/packages/prompts/promptfoo/summary-title/prompt.ts

@@ -0,0 +1,18 @@
+// TypeScript prompt wrapper that uses actual chain implementation
+import { chainSummaryTitle } from '@lobechat/prompts';
+import type { OpenAIChatMessage } from '@lobechat/types';
+
+interface PromptVars {
+  locale: string;
+  messages: OpenAIChatMessage[];
+}
+
+export default function generatePrompt({ vars }: { vars: PromptVars }) {
+  const { messages, locale } = vars;
+
+  // Use the actual chain function from src
+  const result = chainSummaryTitle(messages, locale);
+
+  // Return messages array as expected by promptfoo
+  return result.messages || [];
+}

package/packages/prompts/promptfoo/translate/eval.yaml

@@ -0,0 +1,79 @@
+description: Test translation accuracy between different languages
+
+providers:
+  - openai:chat:gpt-5-mini
+  - openai:chat:claude-3-5-haiku-latest
+  - openai:chat:gemini-flash-latest
+  - openai:chat:deepseek-chat
+
+prompts:
+  - file://promptfoo/translate/prompt.ts
+
+tests:
+  - vars:
+      content: 'Hello, how are you?'
+      from: 'en-US'
+      to: 'zh-CN'
+    assert:
+      - type: contains-any
+        value: ['你好', '您好']
+      - type: not-contains
+        value: 'Hello'
+
+  - vars:
+      content: '你好，你怎么样？'
+      from: 'zh-CN'
+      to: 'en-US'
+    assert:
+      - type: contains-any
+        value: ['Hello', 'Hi', 'how are you', 'How are you']
+      - type: not-contains
+        value: '你好'
+
+  - vars:
+      content: 'Je suis content de vous rencontrer'
+      from: 'fr-FR'
+      to: 'en-US'
+    assert:
+      - type: contains-any
+        value: ['pleased', 'happy', 'glad', 'meet', 'meeting']
+      - type: not-contains
+        value: 'Je suis'
+
+  - vars:
+      content: 'The weather is beautiful today'
+      from: 'en-US'
+      to: 'es-ES'
+    assert:
+      - type: contains-any
+        value: ['tiempo', 'clima', 'hermoso', 'bonito', 'hoy', 'día', 'precioso']
+      - type: not-contains
+        value: 'weather'
+
+  - vars:
+      content: 'I love programming with TypeScript'
+      from: 'en-US'
+      to: 'ja-JP'
+    assert:
+      - type: contains
+        value: 'TypeScript' # Technical terms often remain unchanged
+      - type: not-contains
+        value: 'I love'
+
+  - vars:
+      content: 'Machine learning is revolutionizing technology'
+      from: 'en-US'
+      to: 'de-DE'
+    assert:
+      - type: contains-any
+        value: ['Technologie', 'revolutioniert', 'maschinelles', 'Lernen']
+      - type: not-contains
+        value: 'Machine learning'
+
+  - vars:
+      content: 'API_KEY_12345'
+      from: 'en-US'
+      to: 'zh-CN'
+    assert:
+      - type: contains
+        value: 'API_KEY_12345'

package/packages/prompts/promptfoo/translate/prompt.ts

@@ -0,0 +1,18 @@
+// TypeScript prompt wrapper that uses actual chain implementation
+import { chainTranslate } from '@lobechat/prompts';
+
+interface PromptVars {
+  content: string;
+  from: string;
+  to: string;
+}
+
+export default function generatePrompt({ vars }: { vars: PromptVars }) {
+  const { content, to } = vars;
+
+  // Use the actual chain function from src
+  const result = chainTranslate(content, to);
+
+  // Return messages array as expected by promptfoo
+  return result.messages || [];
+}

package/packages/prompts/promptfooconfig.yaml

@@ -0,0 +1,35 @@
+description: LobeHub Prompts Testing Suite
+
+# Test configurations - run all prompt tests
+testPaths:
+  - promptfoo/translate/eval.yaml
+  - promptfoo/summary-title/eval.yaml
+  - promptfoo/language-detection/eval.yaml
+  - promptfoo/emoji-picker/eval.yaml
+  - promptfoo/knowledge-qa/eval.yaml
+
+# Output configuration
+outputPath: promptfoo-results.json
+
+# Default test settings
+defaultTest:
+  assert:
+    - type: llm-rubric
+      provider: openai:gpt-5-mini
+      value: "The response should be relevant and well-formatted"
+    - type: cost
+      threshold: 0.01  # Maximum cost per test in USD
+
+# Environment variables for API keys
+env:
+  OPENAI_API_KEY: ${OPENAI_API_KEY}
+  ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY}
+
+# Evaluation settings
+evaluateOptions:
+  maxConcurrency: 5
+  delay: 100
+
+# TypeScript support
+transforms:
+  - "typescript"