@lobehub/lobehub 2.0.0-next.37 → 2.0.0-next.39

package/packages/context-engine/src/processors/__tests__/MessageCleanup.test.ts

@@ -131,6 +131,34 @@ describe('MessageCleanupProcessor', () => {
       });
     });
 
+    it('should preserve reasoning in assistant messages', async () => {
+      const processor = new MessageCleanupProcessor();
+      const reasoning = {
+        content: 'Let me think about this...',
+        signature: 'sha256:abc123',
+      };
+
+      const context = createContext([
+        {
+          content: 'Here is the answer',
+          extraField: 'remove',
+          id: 'msg5',
+          reasoning: reasoning,
+          role: 'assistant',
+          timestamp: Date.now(),
+        },
+      ]);
+
+      const result = await processor.process(context);
+
+      expect(result.messages).toHaveLength(1);
+      expect(result.messages[0]).toEqual({
+        content: 'Here is the answer',
+        reasoning: reasoning,
+        role: 'assistant',
+      });
+    });
+
     it('should clean tool messages with name', async () => {
       const processor = new MessageCleanupProcessor();
       const context = createContext([

package/packages/obervability-otel/package.json

@@ -3,7 +3,9 @@
   "version": "1.0.0",
   "private": true,
   "exports": {
-    "./node": "./src/node.ts"
+    "./node": "./src/node.ts",
+    "./api": "./src/api.ts",
+    "./trpc": "./src/trpc/index.ts"
   },
   "dependencies": {
     "@opentelemetry/api": "^1.9.0",

package/packages/obervability-otel/src/api.ts

@@ -0,0 +1,2 @@
+export type { Attributes, Span } from '@opentelemetry/api';
+export { diag, metrics, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api';

package/packages/obervability-otel/src/trpc/convention.ts

@@ -0,0 +1,16 @@
+export enum TRPCAttribute {
+  RPC_METHOD = 'rpc.method',
+  RPC_SERVICE = 'rpc.service',
+  RPC_SYSTEM = 'rpc.system',
+  RPC_TRPC_PATH = 'rpc.trpc.path',
+  RPC_TRPC_STATUS_CODE = 'rpc.trpc.status_code',
+  RPC_TRPC_SUCCESS = 'rpc.trpc.success',
+  RPC_TRPC_TYPE = 'rpc.trpc.type',
+}
+
+export {
+  ATTR_ERROR_TYPE,
+  ATTR_EXCEPTION_MESSAGE,
+  ATTR_EXCEPTION_STACKTRACE,
+  ATTR_EXCEPTION_TYPE,
+} from '@opentelemetry/semantic-conventions';

package/packages/obervability-otel/src/trpc/index.test.ts

@@ -0,0 +1,38 @@
+import { parseTRPCPath, tRPCConventionFromPathAndType } from './';
+
+describe('splitRpcPath', () => {
+  it('should split path into service and method with url', () => {
+    const { method, service } = parseTRPCPath('/trpc/lambda/someService.someMethod');
+    expect(method).toBe('someMethod');
+    expect(service).toBe('someService');
+  });
+
+  it('should split path into service and method without url', () => {
+    const { method, service } = parseTRPCPath('someService.someMethod');
+    expect(method).toBe('someMethod');
+    expect(service).toBe('someService');
+  });
+});
+
+describe('createBaseAttributes', () => {
+  it('should create base attributes with service and method', () => {
+    const attributes = tRPCConventionFromPathAndType('someService.someMethod', 'query');
+    expect(attributes).toEqual({
+      'rpc.system': 'trpc',
+      'rpc.trpc.path': 'someService.someMethod',
+      'rpc.trpc.type': 'query',
+      'rpc.service': 'someService',
+      'rpc.method': 'someMethod',
+    });
+  });
+
+  it('should create base attributes without service', () => {
+    const attributes = tRPCConventionFromPathAndType('someMethod', 'mutation');
+    expect(attributes).toEqual({
+      'rpc.system': 'trpc',
+      'rpc.trpc.path': 'someMethod',
+      'rpc.trpc.type': 'mutation',
+      'rpc.method': 'someMethod',
+    });
+  });
+});

package/packages/obervability-otel/src/trpc/index.ts

@@ -0,0 +1,62 @@
+import { Attributes } from '@opentelemetry/api';
+
+import { TRPCAttribute } from './convention';
+
+export const DEFAULT_ERROR_CODE = 'UNKNOWN_ERROR';
+export const DEFAULT_SUCCESS_STATUS = 'OK';
+
+const textEncoder = new TextEncoder();
+
+export function parseTRPCPath(path: string) {
+  const parts = path?.split('.') ?? [];
+  if (parts.length <= 1) {
+    return { method: path, service: undefined };
+  }
+
+  const method = parts.at(-1);
+  const service = parts.slice(0, -1).join('.').split('/').pop();
+  return { method, service };
+}
+
+export function tRPCConventionFromPathAndType(path: string, type: string): Attributes {
+  const { method, service } = parseTRPCPath(path);
+
+  const attributes: Attributes = {
+    [TRPCAttribute.RPC_SYSTEM]: 'trpc',
+    [TRPCAttribute.RPC_TRPC_PATH]: path,
+    [TRPCAttribute.RPC_TRPC_TYPE]: type,
+  };
+
+  if (service) {
+    attributes[TRPCAttribute.RPC_SERVICE] = service;
+  }
+  if (method) {
+    attributes[TRPCAttribute.RPC_METHOD] = method;
+  }
+
+  return attributes;
+}
+
+export const getPayloadSize = (payload: unknown): number | undefined => {
+  if (payload === undefined || payload === null) return undefined;
+
+  try {
+    const serialized = JSON.stringify(payload);
+    return textEncoder.encode(serialized).length;
+  } catch {
+    return undefined;
+  }
+};
+
+export const createAttributesForMetrics = (
+  baseAttributes: Attributes,
+  statusCode: string,
+  extraAttributes?: Attributes,
+): Attributes => ({
+  ...baseAttributes,
+  [TRPCAttribute.RPC_TRPC_STATUS_CODE]: statusCode,
+  ...extraAttributes,
+});
+
+export * from './convention';
+export * from './metrics';

package/packages/obervability-otel/src/trpc/metrics.ts

@@ -0,0 +1,31 @@
+import { metrics } from '@opentelemetry/api';
+
+const meter = metrics.getMeter('trpc-server');
+
+export const serverDurationHistogram = meter.createHistogram('rpc.server.duration', {
+  description: 'Measures the duration of inbound RPC.',
+  unit: 'ms',
+});
+
+export const serverRequestSizeHistogram = meter.createHistogram('rpc.server.request.size', {
+  description: 'Measures the size of RPC request messages (uncompressed).',
+  unit: 'By',
+});
+
+export const serverResponseSizeHistogram = meter.createHistogram('rpc.server.response.size', {
+  description: 'Measures the size of RPC response messages (uncompressed).',
+  unit: 'By',
+});
+
+export const serverRequestsPerRpcHistogram = meter.createHistogram('rpc.server.requests_per_rpc', {
+  description: 'Measures the number of messages received per RPC.',
+  unit: '{count}',
+});
+
+export const serverResponsesPerRpcHistogram = meter.createHistogram(
+  'rpc.server.responses_per_rpc',
+  {
+    description: 'Measures the number of messages sent per RPC.',
+    unit: '{count}',
+  },
+);

package/packages/types/src/usage/usageRecord.ts

@@ -0,0 +1,54 @@
+import { MessageMetadata } from '../message';
+
+export interface UsageRecordItem {
+  createdAt: Date;
+  /**
+   * ID
+   **/
+  id: string;
+  inputStartAt?: Date | null;
+  /**
+   * Meta information
+   **/
+  metadata?: MessageMetadata | null;
+  /**
+   * Model id
+   */
+  model: string;
+  outputFinishAt?: Date | null;
+  outputStartAt?: Date | null;
+  /**
+   * Provider id
+   */
+  provider: string;
+  /**
+   * Spend
+   **/
+  spend: number;
+  /**
+   * Usage details
+   **/
+  totalInputTokens?: number | null;
+  totalOutputTokens?: number | null;
+  totalTokens?: number | null;
+  /**
+   * Performance details
+   **/
+  tps?: number | null;
+  ttft?: number | null;
+  /**
+   * Call types
+   **/
+  type: string;
+  updatedAt: Date;
+  userId: string;
+}
+
+export type UsageLog = {
+  date: number;
+  day: string;
+  records: UsageRecordItem[];
+  totalRequests: number;
+  totalSpend: number;
+  totalTokens: number;
+};

package/packages/web-crawler/src/crawImpl/browserless.ts

@@ -56,7 +56,7 @@ export const browserless: CrawlImpl = async (url, { filterOptions }) => {
     if (
       !!result.content &&
       result.title &&
-      // Just a moment... 说明被 CF 拦截了
+      // "Just a moment..." indicates being blocked by CloudFlare
       result.title.trim() !== 'Just a moment...'
     ) {
       return {

package/packages/web-crawler/src/crawImpl/naive.ts

@@ -6,25 +6,25 @@ import { htmlToMarkdown } from '../utils/htmlToMarkdown';
 import { DEFAULT_TIMEOUT, withTimeout } from '../utils/withTimeout';
 
 const mixinHeaders = {
-  // 接受的内容类型
+  // Accepted content types
   'Accept':
     'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
-  // 接受的编码方式
+  // Accepted encoding methods
   'Accept-Encoding': 'gzip, deflate, br',
-  // 接受的语言
+  // Accepted languages
   'Accept-Language': 'en-US,en;q=0.9,zh;q=0.8',
-  // 缓存控制
+  // Cache control
   'Cache-Control': 'max-age=0',
-  // 连接类型
+  // Connection type
   'Connection': 'keep-alive',
-  // 表明请求来自哪个站点
+  // Indicates which site the request is from
   'Referer': 'https://www.google.com/',
-  // 升级不安全请求
+  // Upgrade insecure requests
   'Upgrade-Insecure-Requests': '1',
-  // 模拟真实浏览器的 User-Agent
+  // Simulate real browser User-Agent
   'User-Agent':
     'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
-  // 防止跨站请求伪造
+  // Prevent cross-site request forgery
   'sec-ch-ua': '"Google Chrome";v="121", "Not A(Brand";v="99", "Chromium";v="121"',
   'sec-ch-ua-mobile': '?0',
   'sec-ch-ua-platform': '"Windows"',

package/packages/web-crawler/src/crawler.ts

@@ -19,8 +19,8 @@ export class Crawler {
   }
 
   /**
-   * 爬取网页内容
-   * @param options 爬取选项
+   * Crawl webpage content
+   * @param options Crawl options
    */
   async crawl({
     url,
@@ -31,14 +31,14 @@ export class Crawler {
     impls?: CrawlImplType[];
     url: string;
   }): Promise<CrawlUniformResult> {
-    // 应用URL规则
+    // Apply URL rules
     const {
       transformedUrl,
       filterOptions: ruleFilterOptions,
       impls: ruleImpls,
     } = applyUrlRules(url, crawUrlRules);
 
-    // 合并用户提供的过滤选项和规则中的过滤选项，用户选项优先
+    // Merge user-provided filter options and rule filter options, user options take priority
     const mergedFilterOptions = {
       ...ruleFilterOptions,
       ...userFilterOptions,
@@ -53,7 +53,7 @@ export class Crawler {
       ? (userImpls.filter((impl) => Object.keys(crawlImpls).includes(impl)) as CrawlImplType[])
       : systemImpls;
 
-    //   按照内置的实现顺序依次尝试
+    // Try each implementation in the built-in order
     for (const impl of finalImpls) {
       try {
         const res = await crawlImpls[impl](transformedUrl, { filterOptions: mergedFilterOptions });

package/packages/web-crawler/src/urlRules.ts

@@ -1,32 +1,32 @@
 import { CrawlUrlRule } from './type';
 
 export const crawUrlRules: CrawlUrlRule[] = [
-  // 搜狗微信链接，使用 search1api
+  // Sogou WeChat links, use search1api
   {
     impls: ['search1api'],
     urlPattern: 'https://weixin.sogou.com/link(.*)',
   },
-  // 搜狗链接，使用 search1api
+  // Sogou links, use search1api
   {
     impls: ['search1api'],
     urlPattern: 'https://sogou.com/link(.*)',
   },
-  // YouTube 链接，使用 search1api，格式化 markdown，且可以返回字幕内容
+  // YouTube links, use search1api, formatted as markdown, can return subtitle content
   {
     impls: ['search1api'],
     urlPattern: 'https://www.youtube.com/watch(.*)',
   },
-  // Reddit 链接，使用 search1api，格式化 markdown，包含标题、作者、互动数量、具体评论内容等
+  // Reddit links, use search1api, formatted as markdown, includes title, author, interaction count, specific comment content, etc.
   {
     impls: ['search1api'],
     urlPattern: 'https://www.reddit.com/r/(.*)/comments/(.*)',
   },
-  // 微信公众号有爬虫防护，优先使用 search1api，jina 作为兜底（目前 jina 爬取会被风控）
+  // WeChat official accounts have crawler protection, use search1api first, jina as fallback (currently jina crawling may be blocked)
   {
     impls: ['search1api', 'jina'],
     urlPattern: 'https://mp.weixin.qq.com(.*)',
   },
-  // github 源码解析
+  // GitHub source code parsing
   {
     filterOptions: {
       enableReadability: false,
@@ -43,7 +43,7 @@ export const crawUrlRules: CrawlUrlRule[] = [
     // GitHub discussion
     urlPattern: 'https://github.com/(.*)/discussions/(.*)',
   },
-  // 所有 PDF 都用 jina
+  // All PDFs use jina
   {
     impls: ['jina'],
     urlPattern: 'https://(.*).pdf',
@@ -53,7 +53,7 @@ export const crawUrlRules: CrawlUrlRule[] = [
     impls: ['jina'],
     urlPattern: 'https://arxiv.org/pdf/(.*)',
   },
-  // 知乎有爬虫防护，使用 jina
+  // Zhihu has crawler protection, use jina
   {
     impls: ['jina'],
     urlPattern: 'https://zhuanlan.zhihu.com(.*)',
@@ -63,7 +63,7 @@ export const crawUrlRules: CrawlUrlRule[] = [
     urlPattern: 'https://zhihu.com(.*)',
   },
   {
-    // Medium 文章转换为 Scribe.rip
+    // Convert Medium articles to Scribe.rip
     urlPattern: 'https://medium.com/(.*)',
     urlTransform: 'https://scribe.rip/$1',
   },
@@ -74,10 +74,10 @@ export const crawUrlRules: CrawlUrlRule[] = [
     impls: ['jina', 'browserless'],
     urlPattern: 'https://(twitter.com|x.com)/(.*)',
   },
-  // 体育数据网站规则
+  // Sports data website rules
   {
     filterOptions: {
-      // 对体育数据表格禁用 Readability 并且转换为纯文本
+      // Disable Readability for sports data tables and convert to plain text
       enableReadability: false,
       pureText: true,
     },
@@ -94,13 +94,13 @@ export const crawUrlRules: CrawlUrlRule[] = [
     impls: ['jina'],
     urlPattern: 'https://cvpr.thecvf.com(.*)',
   },
-  // 飞书用 jina
+  // Feishu use jina
   // https://github.com/lobehub/lobe-chat/issues/6879
   {
     impls: ['jina'],
     urlPattern: 'https://(.*).feishu.cn/(.*)',
   },
-  // 小红书存在爬虫防护，使用 Search1API 或 Jina (备用)
+  // Xiaohongshu has crawler protection, use Search1API or Jina (fallback)
   {
     impls: ['search1api', 'jina'],
     urlPattern: 'https://(.*).xiaohongshu.com/(.*)',

package/packages/web-crawler/src/utils/appUrlRules.ts

@@ -9,14 +9,14 @@ export const applyUrlRules = (
   transformedUrl: string;
 } => {
   for (const rule of urlRules) {
-    // 转换为正则表达式
+    // Convert to regular expression
     const regex = new RegExp(rule.urlPattern);
     const match = url.match(regex);
 
     if (match) {
       if (rule.urlTransform) {
-        // 如果有转换规则，进行 URL 转换
-        // 替换 $1, $2 等占位符为捕获组内容
+        // If there is a transformation rule, perform URL transformation
+        // Replace placeholders like $1, $2 with capture group content
         const transformedUrl = rule.urlTransform.replaceAll(
           /\$(\d+)/g,
           (_, index) => match[parseInt(index)] || '',
@@ -28,7 +28,7 @@ export const applyUrlRules = (
           transformedUrl,
         };
       } else {
-        // 没有转换规则但匹配了模式，只返回过滤选项
+        // No transformation rule but pattern matched, only return filter options
         return {
           filterOptions: rule.filterOptions,
           impls: rule.impls,
@@ -38,6 +38,6 @@ export const applyUrlRules = (
     }
   }
 
-  // 没有匹配任何规则，返回原始 URL
+  // No rule matched, return original URL
   return { transformedUrl: url };
 };

package/src/app/[variants]/(main)/profile/hooks/useCategory.tsx

@@ -1,5 +1,5 @@
 import { Icon } from '@lobehub/ui';
-import { ChartColumnBigIcon, KeyIcon, ShieldCheck, UserCircle } from 'lucide-react';
+import { BadgeCentIcon, ChartColumnBigIcon, KeyIcon, ShieldCheck, UserCircle } from 'lucide-react';
 import Link from 'next/link';
 import { useTranslation } from 'react-i18next';
 
@@ -54,6 +54,15 @@ export const useCategory = () => {
         </Link>
       ),
     },
+    {
+      icon: <Icon icon={BadgeCentIcon} />,
+      key: ProfileTabs.Usage,
+      label: (
+        <Link href={'/profile/usage'} onClick={(e) => e.preventDefault()}>
+          {t('tab.usage')}
+        </Link>
+      ),
+    },
   ].filter(Boolean) as MenuProps['items'];
 
   return cateItems;

package/src/app/[variants]/(main)/profile/usage/Client.tsx

@@ -0,0 +1,114 @@
+'use client';
+
+import { Icon, Segmented } from '@lobehub/ui';
+import { Col, DatePicker, DatePickerProps, Row } from 'antd';
+import dayjs from 'dayjs';
+import { Brain, Codesandbox } from 'lucide-react';
+import { memo, useEffect, useState } from 'react';
+import { useTranslation } from 'react-i18next';
+import { Flexbox } from 'react-layout-kit';
+
+import { useClientDataSWR } from '@/libs/swr';
+import { usageService } from '@/services/usage';
+import { UsageLog } from '@/types/usage/usageRecord';
+
+import Welcome from '../stats/features/Welcome';
+import UsageCards from './features/UsageCards';
+import UsageTable from './features/UsageTable';
+import UsageTrends from './features/UsageTrends';
+
+export interface UsageChartProps {
+  data?: UsageLog[];
+  dateStrings?: string;
+  groupBy?: GroupBy;
+  inShare?: boolean;
+  isLoading?: boolean;
+  mobile?: boolean;
+}
+
+export enum GroupBy {
+  Model = 'model',
+  Provider = 'provider',
+}
+
+const Client = memo<{ mobile?: boolean }>(({ mobile }) => {
+  const { t, i18n } = useTranslation('auth');
+  dayjs.locale(i18n.language);
+
+  const [groupBy, setGroupBy] = useState<GroupBy>(GroupBy.Model);
+  const [dateRange, setDateRange] = useState<dayjs.Dayjs>(dayjs(new Date()));
+  const [dateStrings, setDateStrings] = useState<string>();
+
+  const { data, isLoading, mutate } = useClientDataSWR('usage-stat', async () =>
+    usageService.findAndGroupByDay(dateStrings),
+  );
+
+  useEffect(() => {
+    if (dateStrings) {
+      mutate();
+    }
+  }, [dateStrings]);
+
+  const handleDateChange: DatePickerProps['onChange'] = (dates, dateStrings) => {
+    setDateRange(dates);
+    if (typeof dateStrings === 'string') {
+      setDateStrings(dateStrings);
+    }
+  };
+
+  return (
+    <Flexbox gap={mobile ? 0 : 24}>
+      <Flexbox>
+        <Row>
+          <Col span={16}>
+            {mobile ? (
+              <Welcome mobile />
+            ) : (
+              <Flexbox align={'flex-start'} gap={16} horizontal justify={'space-between'}>
+                <Welcome />
+              </Flexbox>
+            )}
+          </Col>
+          <Col span={8}>
+            <Flexbox gap={16} horizontal>
+              <Segmented
+                onChange={(v) => setGroupBy(v as GroupBy)}
+                options={[
+                  {
+                    icon: <Icon icon={Codesandbox} />,
+                    label: t('usage.welcome.model'),
+                    value: GroupBy.Model,
+                  },
+                  {
+                    icon: <Icon icon={Brain} />,
+                    label: t('usage.welcome.provider'),
+                    value: GroupBy.Provider,
+                  },
+                ]}
+                value={groupBy}
+              />
+              <DatePicker onChange={handleDateChange} picker="month" value={dateRange} />
+            </Flexbox>
+          </Col>
+        </Row>
+      </Flexbox>
+      <Flexbox>
+        <UsageCards data={data} groupBy={groupBy} isLoading={isLoading} />
+      </Flexbox>
+      <Flexbox>
+        <Row gutter={[16, 16]}>
+          <Col span={24}>
+            <UsageTrends data={data} groupBy={groupBy} isLoading={isLoading} />
+          </Col>
+        </Row>
+      </Flexbox>
+      <Row>
+        <Col span={24}>
+          <UsageTable dateStrings={dateStrings} />
+        </Col>
+      </Row>
+    </Flexbox>
+  );
+});
+
+export default Client;