@lobehub/lobehub 2.0.0-next.4 → 2.0.0-next.40

package/packages/web-crawler/src/crawImpl/firecrawl.ts

@@ -3,36 +3,55 @@ import { NetworkConnectionError, PageNotFoundError, TimeoutError } from '../util
 import { DEFAULT_TIMEOUT, withTimeout } from '../utils/withTimeout';
 
 interface FirecrawlMetadata {
-  description: string;
-  keywords: string;
-  language: string;
+  description?: string;
+  error?: string;
+  keywords?: string;
+  language?: string;
   ogDescription?: string;
   ogImage?: string;
   ogLocaleAlternate?: string[];
   ogSiteName?: string;
   ogTitle?: string;
   ogUrl?: string;
-  robots: string;
-  statusCode: number;
+  robots?: string;
   sourceURL: string;
-  title: string;
+  statusCode: number;
+  title?: string;
 }
 
 interface FirecrawlResults {
+  actions?: {
+    javascriptReturns?: Array<{ type: string; value: any }>;
+    pdfs?: string[];
+    scrapes?: Array<{ html: string; url: string }>;
+    screenshots?: string[];
+  };
+  changeTracking?: {
+    changeStatus?: string;
+    diff?: string;
+    json?: Record<string, any>;
+    previousScrapeAt?: string;
+    visibility?: string;
+  };
   html?: string;
+  links?: string[];
   markdown?: string;
   metadata: FirecrawlMetadata;
+  rawHtml?: string;
+  screenshot?: string;
+  summary?: string;
+  warning?: string;
 }
 
 interface FirecrawlResponse {
-  success: boolean;
   data: FirecrawlResults;
+  success: boolean;
 }
 
 export const firecrawl: CrawlImpl = async (url) => {
   // Get API key from environment variable
   const apiKey = process.env.FIRECRAWL_API_KEY;
-  const baseUrl = process.env.FIRECRAWL_URL || 'https://api.firecrawl.dev/v1';
+  const baseUrl = process.env.FIRECRAWL_URL || 'https://api.firecrawl.dev/v2';
 
   let res: Response;
 
@@ -40,7 +59,7 @@ export const firecrawl: CrawlImpl = async (url) => {
     res = await withTimeout(
       fetch(`${baseUrl}/scrape`, {
         body: JSON.stringify({
-          formats: ["markdown"], // ["markdown", "html"]
+          formats: ['markdown'], // ["markdown", "html"]
           url,
         }),
         headers: {
@@ -75,6 +94,14 @@ export const firecrawl: CrawlImpl = async (url) => {
   try {
     const data = (await res.json()) as FirecrawlResponse;
 
+    if (data.data.warning) {
+      console.warn('[Firecrawl] Warning:', data.data.warning);
+    }
+
+    if (data.data.metadata.error) {
+      console.error('[Firecrawl] Metadata error:', data.data.metadata.error);
+    }
+
     // Check if content is empty or too short
     if (!data.data.markdown || data.data.markdown.length < 100) {
       return;
@@ -83,14 +110,14 @@ export const firecrawl: CrawlImpl = async (url) => {
     return {
       content: data.data.markdown,
       contentType: 'text',
-      description: data.data.metadata.description,
+      description: data.data.metadata.description || '',
       length: data.data.markdown.length,
       siteName: new URL(url).hostname,
-      title: data.data.metadata.title,
+      title: data.data.metadata.title || '',
       url: url,
     } satisfies CrawlSuccessResult;
   } catch (error) {
-    console.error(error);
+    console.error('[Firecrawl] Parse error:', error);
   }
 
   return;

package/packages/web-crawler/src/crawImpl/naive.ts

@@ -6,25 +6,25 @@ import { htmlToMarkdown } from '../utils/htmlToMarkdown';
 import { DEFAULT_TIMEOUT, withTimeout } from '../utils/withTimeout';
 
 const mixinHeaders = {
-  // 接受的内容类型
+  // Accepted content types
   'Accept':
     'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
-  // 接受的编码方式
+  // Accepted encoding methods
   'Accept-Encoding': 'gzip, deflate, br',
-  // 接受的语言
+  // Accepted languages
   'Accept-Language': 'en-US,en;q=0.9,zh;q=0.8',
-  // 缓存控制
+  // Cache control
   'Cache-Control': 'max-age=0',
-  // 连接类型
+  // Connection type
   'Connection': 'keep-alive',
-  // 表明请求来自哪个站点
+  // Indicates which site the request is from
   'Referer': 'https://www.google.com/',
-  // 升级不安全请求
+  // Upgrade insecure requests
   'Upgrade-Insecure-Requests': '1',
-  // 模拟真实浏览器的 User-Agent
+  // Simulate real browser User-Agent
   'User-Agent':
     'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
-  // 防止跨站请求伪造
+  // Prevent cross-site request forgery
   'sec-ch-ua': '"Google Chrome";v="121", "Not A(Brand";v="99", "Chromium";v="121"',
   'sec-ch-ua-mobile': '?0',
   'sec-ch-ua-platform': '"Windows"',

package/packages/web-crawler/src/crawler.ts

@@ -19,8 +19,8 @@ export class Crawler {
   }
 
   /**
-   * 爬取网页内容
-   * @param options 爬取选项
+   * Crawl webpage content
+   * @param options Crawl options
    */
   async crawl({
     url,
@@ -31,14 +31,14 @@ export class Crawler {
     impls?: CrawlImplType[];
     url: string;
   }): Promise<CrawlUniformResult> {
-    // 应用URL规则
+    // Apply URL rules
     const {
       transformedUrl,
       filterOptions: ruleFilterOptions,
       impls: ruleImpls,
     } = applyUrlRules(url, crawUrlRules);
 
-    // 合并用户提供的过滤选项和规则中的过滤选项，用户选项优先
+    // Merge user-provided filter options and rule filter options, user options take priority
     const mergedFilterOptions = {
       ...ruleFilterOptions,
       ...userFilterOptions,
@@ -53,7 +53,7 @@ export class Crawler {
       ? (userImpls.filter((impl) => Object.keys(crawlImpls).includes(impl)) as CrawlImplType[])
       : systemImpls;
 
-    //   按照内置的实现顺序依次尝试
+    // Try each implementation in the built-in order
     for (const impl of finalImpls) {
       try {
         const res = await crawlImpls[impl](transformedUrl, { filterOptions: mergedFilterOptions });

package/packages/web-crawler/src/urlRules.ts

@@ -1,32 +1,32 @@
 import { CrawlUrlRule } from './type';
 
 export const crawUrlRules: CrawlUrlRule[] = [
-  // 搜狗微信链接，使用 search1api
+  // Sogou WeChat links, use search1api
   {
     impls: ['search1api'],
     urlPattern: 'https://weixin.sogou.com/link(.*)',
   },
-  // 搜狗链接，使用 search1api
+  // Sogou links, use search1api
   {
     impls: ['search1api'],
     urlPattern: 'https://sogou.com/link(.*)',
   },
-  // YouTube 链接，使用 search1api，格式化 markdown，且可以返回字幕内容
+  // YouTube links, use search1api, formatted as markdown, can return subtitle content
   {
     impls: ['search1api'],
     urlPattern: 'https://www.youtube.com/watch(.*)',
   },
-  // Reddit 链接，使用 search1api，格式化 markdown，包含标题、作者、互动数量、具体评论内容等
+  // Reddit links, use search1api, formatted as markdown, includes title, author, interaction count, specific comment content, etc.
   {
     impls: ['search1api'],
     urlPattern: 'https://www.reddit.com/r/(.*)/comments/(.*)',
   },
-  // 微信公众号有爬虫防护，优先使用 search1api，jina 作为兜底（目前 jina 爬取会被风控）
+  // WeChat official accounts have crawler protection, use search1api first, jina as fallback (currently jina crawling may be blocked)
   {
     impls: ['search1api', 'jina'],
     urlPattern: 'https://mp.weixin.qq.com(.*)',
   },
-  // github 源码解析
+  // GitHub source code parsing
   {
     filterOptions: {
       enableReadability: false,
@@ -43,7 +43,7 @@ export const crawUrlRules: CrawlUrlRule[] = [
     // GitHub discussion
     urlPattern: 'https://github.com/(.*)/discussions/(.*)',
   },
-  // 所有 PDF 都用 jina
+  // All PDFs use jina
   {
     impls: ['jina'],
     urlPattern: 'https://(.*).pdf',
@@ -53,7 +53,7 @@ export const crawUrlRules: CrawlUrlRule[] = [
     impls: ['jina'],
     urlPattern: 'https://arxiv.org/pdf/(.*)',
   },
-  // 知乎有爬虫防护，使用 jina
+  // Zhihu has crawler protection, use jina
   {
     impls: ['jina'],
     urlPattern: 'https://zhuanlan.zhihu.com(.*)',
@@ -63,7 +63,7 @@ export const crawUrlRules: CrawlUrlRule[] = [
     urlPattern: 'https://zhihu.com(.*)',
   },
   {
-    // Medium 文章转换为 Scribe.rip
+    // Convert Medium articles to Scribe.rip
     urlPattern: 'https://medium.com/(.*)',
     urlTransform: 'https://scribe.rip/$1',
   },
@@ -74,10 +74,10 @@ export const crawUrlRules: CrawlUrlRule[] = [
     impls: ['jina', 'browserless'],
     urlPattern: 'https://(twitter.com|x.com)/(.*)',
   },
-  // 体育数据网站规则
+  // Sports data website rules
   {
     filterOptions: {
-      // 对体育数据表格禁用 Readability 并且转换为纯文本
+      // Disable Readability for sports data tables and convert to plain text
       enableReadability: false,
       pureText: true,
     },
@@ -94,13 +94,13 @@ export const crawUrlRules: CrawlUrlRule[] = [
     impls: ['jina'],
     urlPattern: 'https://cvpr.thecvf.com(.*)',
   },
-  // 飞书用 jina
+  // Feishu use jina
   // https://github.com/lobehub/lobe-chat/issues/6879
   {
     impls: ['jina'],
     urlPattern: 'https://(.*).feishu.cn/(.*)',
   },
-  // 小红书存在爬虫防护，使用 Search1API 或 Jina (备用)
+  // Xiaohongshu has crawler protection, use Search1API or Jina (fallback)
   {
     impls: ['search1api', 'jina'],
     urlPattern: 'https://(.*).xiaohongshu.com/(.*)',

package/packages/web-crawler/src/utils/appUrlRules.ts

@@ -9,14 +9,14 @@ export const applyUrlRules = (
   transformedUrl: string;
 } => {
   for (const rule of urlRules) {
-    // 转换为正则表达式
+    // Convert to regular expression
     const regex = new RegExp(rule.urlPattern);
     const match = url.match(regex);
 
     if (match) {
       if (rule.urlTransform) {
-        // 如果有转换规则，进行 URL 转换
-        // 替换 $1, $2 等占位符为捕获组内容
+        // If there is a transformation rule, perform URL transformation
+        // Replace placeholders like $1, $2 with capture group content
         const transformedUrl = rule.urlTransform.replaceAll(
           /\$(\d+)/g,
           (_, index) => match[parseInt(index)] || '',
@@ -28,7 +28,7 @@ export const applyUrlRules = (
           transformedUrl,
         };
       } else {
-        // 没有转换规则但匹配了模式，只返回过滤选项
+        // No transformation rule but pattern matched, only return filter options
         return {
           filterOptions: rule.filterOptions,
           impls: rule.impls,
@@ -38,6 +38,6 @@ export const applyUrlRules = (
     }
   }
 
-  // 没有匹配任何规则，返回原始 URL
+  // No rule matched, return original URL
   return { transformedUrl: url };
 };

package/packages/web-crawler/tsconfig.json

@@ -6,7 +6,6 @@
     "sourceMap": true,
     "skipDefaultLibCheck": true,
     "jsx": "react-jsx",
-    "baseUrl": ".",
     "allowSyntheticDefaultImports": true,
     "moduleResolution": "node",
     "forceConsistentCasingInFileNames": true,

package/renovate.json

@@ -1,5 +1,7 @@
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "automerge": false,
+  "dependencyDashboard": true,
   "extends": [
     "config:recommended",
     ":dependencyDashboard",
@@ -9,17 +11,37 @@
     ":semanticPrefixFixDepsChoreOthers",
     "group:monorepos",
     "group:recommended",
-    "group:allNonMajor",
     "replacements:all",
     "workarounds:all"
   ],
   "ignoreDeps": [],
   "labels": ["dependencies"],
+  "packageRules": [
+    // 1) Pinned deps: isolate (OK to use separate* here because there's no matchUpdateTypes)
+    {
+      "description": "Isolate PRs for pinned deps (exact x.y.z)",
+      "matchManagers": ["npm", "pnpm", "yarn", "bun"],
+      "matchCurrentValue": "^\\d+\\.\\d+\\.\\d+([+-][0-9A-Za-z.-]+)?$",
+      "groupName": null,
+      "separateMinorPatch": true,
+      "separateMajorMinor": true
+    },
+    // 2) Non-pinned deps: Patch versions, grouped together
+    {
+      "description": "Group patch versions together for non-pinned deps",
+      "matchManagers": ["npm", "pnpm", "yarn", "bun"],
+      "matchCurrentValue": "/(^[~^]|[<>=| -])/", // anything that looks like a range
+      "groupName": "patch dependencies",
+      "matchUpdateTypes": ["patch"]
+    }
+  ],
   "postUpdateOptions": ["yarnDedupeHighest"],
   "prConcurrentLimit": 30,
   "prHourlyLimit": 0,
   "rangeStrategy": "bump",
   "rebaseWhen": "conflicted",
   "schedule": "on sunday before 6:00am",
+  "separateMajorMinor": true,
+  "separateMultipleMajor": true,
   "timezone": "UTC"
 }

package/scripts/migrateServerDB/errorHint.js

@@ -15,13 +15,7 @@ if you have any other question, please open issue here: https://github.com/lobeh
 const DB_FAIL_INIT_HINT = `------------------------------------------------------------------------------------------
 ⚠️ Database migrate failed due to not find the db instance.
 
-1) You might not switch to server db mode, please set the env blow and try again:
-
-\`\`\`
-NEXT_PUBLIC_SERVICE_MODE=server
-\`\`\`
-
-2) if you are using docker postgres image, you may need to set DATABASE_DRIVER to node
+if you are using docker postgres image, you may need to set DATABASE_DRIVER to node
 
 \`\`\`
 DATABASE_DRIVER=node

package/scripts/migrateServerDB/index.ts

@@ -17,13 +17,14 @@ const isDesktop = process.env.NEXT_PUBLIC_IS_DESKTOP_APP === '1';
 const runMigrations = async () => {
   const { serverDB } = await import('../../packages/database/src/server');
 
+  const time = Date.now();
   if (process.env.DATABASE_DRIVER === 'node') {
     await nodeMigrate(serverDB, { migrationsFolder });
   } else {
     await neonMigrate(serverDB, { migrationsFolder });
   }
 
-  console.log('✅ database migration pass.');
+  console.log('✅ database migration pass. use: %s ms', Date.now() - time);
   // eslint-disable-next-line unicorn/no-process-exit
   process.exit(0);
 };

package/src/app/(backend)/webapi/revalidate/route.ts

@@ -18,7 +18,7 @@ export const GET = async (request: NextRequest) => {
     return NextResponse.json('tag query parameter is required', { status: 400 });
   }
 
-  revalidateTag(tag);
+  revalidateTag(tag, 'max');
 
   return Response.json({ now: Date.now(), revalidated: true });
 };

package/src/app/[variants]/(auth)/signup/[[...signup]]/page.tsx

@@ -1,7 +1,6 @@
 import { SignUp } from '@clerk/nextjs';
-import { notFound, redirect } from 'next/navigation';
+import { notFound } from 'next/navigation';
 
-import { serverFeatureFlags } from '@/config/featureFlags';
 import { enableClerk } from '@/const/auth';
 import { metadataModule } from '@/server/metadata';
 import { translation } from '@/server/translation';
@@ -21,12 +20,6 @@ export const generateMetadata = async (props: DynamicLayoutProps) => {
 const Page = () => {
   if (!enableClerk) return notFound();
 
-  const enableClerkSignUp = serverFeatureFlags().enableClerkSignUp;
-
-  if (!enableClerkSignUp) {
-    redirect('/login');
-  }
-
   return <SignUp path="/signup" />;
 };
 

package/src/app/[variants]/(main)/(mobile)/me/(home)/__tests__/useCategory.test.tsx

@@ -40,6 +40,15 @@ vi.mock('@/const/auth', () => ({
   },
 }));
 
+// Mock version constants
+vi.mock('@/const/version', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('@/const/version')>();
+  return {
+    ...actual,
+    isServerMode: false,
+  };
+});
+
 afterEach(() => {
   enableAuth = true;
   enableClerk = true;

package/src/app/[variants]/(main)/(mobile)/me/(home)/features/UserBanner.tsx

@@ -6,7 +6,6 @@ import { memo } from 'react';
 import { Flexbox } from 'react-layout-kit';
 
 import { enableAuth, enableNextAuth } from '@/const/auth';
-import { isDeprecatedEdition } from '@/const/version';
 import DataStatistics from '@/features/User/DataStatistics';
 import UserInfo from '@/features/User/UserInfo';
 import UserLoginOrSignup from '@/features/User/UserLoginOrSignup/Community';
@@ -25,11 +24,9 @@ const UserBanner = memo(() => {
           <Link href={'/profile'} style={{ color: 'inherit' }}>
             <UserInfo />
           </Link>
-          {!isDeprecatedEdition && (
-            <Link href={'/profile/stats'} style={{ color: 'inherit' }}>
-              <DataStatistics paddingInline={12} />
-            </Link>
-          )}
+          <Link href={'/profile/stats'} style={{ color: 'inherit' }}>
+            <DataStatistics paddingInline={12} />
+          </Link>
         </>
       ) : (
         <UserLoginOrSignup

package/src/app/[variants]/(main)/(mobile)/me/(home)/layout.tsx

@@ -2,7 +2,6 @@ import { PropsWithChildren, Suspense } from 'react';
 
 import Loading from '@/components/Loading/BrandTextLoading';
 import MobileContentLayout from '@/components/server/MobileNavLayout';
-import InitClientDB from '@/features/InitClientDB';
 
 import Header from './features/Header';
 
@@ -10,7 +9,6 @@ const Layout = ({ children }: PropsWithChildren) => {
   return (
     <MobileContentLayout header={<Header />} withNav>
       <Suspense fallback={<Loading />}>{children}</Suspense>
-      <InitClientDB />
     </MobileContentLayout>
   );
 };

package/src/app/[variants]/(main)/_layout/Desktop/SideBar/TopActions.tsx

@@ -32,9 +32,6 @@ const TopActions = memo<TopActionProps>(({ tab, isPinned }) => {
     useServerConfigStore(featureFlagsSelectors);
   const hotkey = useUserStore(settingsSelectors.getHotkeyById(HotkeyEnum.NavigateToChat));
 
-  // Check if server mode is enabled
-  const isServerMode = process.env.NEXT_PUBLIC_SERVICE_MODE === 'server';
-
   const isChatActive = tab === SidebarTabKey.Chat && !isPinned;
   const isFilesActive = tab === SidebarTabKey.Files;
   const isDiscoverActive = tab === SidebarTabKey.Discover;
@@ -69,7 +66,7 @@ const TopActions = memo<TopActionProps>(({ tab, isPinned }) => {
           tooltipProps={{ placement: 'right' }}
         />
       </Link>
-      {enableKnowledgeBase && isServerMode && (
+      {enableKnowledgeBase && (
         <Link aria-label={t('tab.knowledgeBase')} href={'/knowledge'}>
           <ActionIcon
             active={isFilesActive}

package/src/app/[variants]/(main)/chat/ChatRouter.tsx

@@ -0,0 +1,83 @@
+'use client';
+
+import { memo, useEffect } from 'react';
+import { useMediaQuery } from 'react-responsive';
+import { MemoryRouter, Navigate, Route, Routes, useLocation, useNavigate } from 'react-router-dom';
+
+import MainChatPage from './components/MainChatPage';
+import SettingsPage from './components/SettingsPage';
+
+// Get initial path from URL
+const getInitialPath = () => {
+  if (typeof window === 'undefined') return '/';
+  const fullPath = window.location.pathname;
+  const searchParams = window.location.search;
+  const chatIndex = fullPath.indexOf('/chat');
+
+  if (chatIndex !== -1) {
+    const pathAfterChat = fullPath.slice(chatIndex + '/chat'.length) || '/';
+    return pathAfterChat + searchParams;
+  }
+  return '/';
+};
+
+// Helper component to sync URL with MemoryRouter
+const UrlSynchronizer = () => {
+  const location = useLocation();
+  const navigate = useNavigate();
+
+  // Sync initial URL
+  useEffect(() => {
+    const fullPath = window.location.pathname;
+    const searchParams = window.location.search;
+    const chatIndex = fullPath.indexOf('/chat');
+
+    if (chatIndex !== -1) {
+      const pathAfterChat = fullPath.slice(chatIndex + '/chat'.length) || '/';
+      const targetPath = pathAfterChat + searchParams;
+
+      if (location.pathname + location.search !== targetPath) {
+        navigate(targetPath, { replace: true });
+      }
+    }
+  }, []);
+
+  // Update browser URL when location changes
+  useEffect(() => {
+    const normalizedPath = location.pathname === '/' ? '' : location.pathname;
+    const newUrl = `/chat${normalizedPath}${location.search}`;
+    if (window.location.pathname + window.location.search !== newUrl) {
+      window.history.replaceState({}, '', newUrl);
+    }
+  }, [location.pathname, location.search]);
+
+  return null;
+};
+
+const ChatRouter = memo(() => {
+  const mobile = useMediaQuery({ maxWidth: 768 });
+  const routes = (
+    <Routes>
+      <Route element={<MainChatPage mobile={true} />} path="/" />
+      <Route element={<SettingsPage mobile={true} />} path="/settings" />
+      <Route element={<Navigate replace to="/" />} path="*" />
+    </Routes>
+  );
+
+  return (
+    <MemoryRouter initialEntries={[getInitialPath()]} initialIndex={0}>
+      <UrlSynchronizer />
+      {mobile ? (
+        // Mobile Layout
+        routes
+      ) : (
+        // Desktop Layout
+        <MainChatPage mobile={false} />
+      )}
+    </MemoryRouter>
+  );
+});
+
+ChatRouter.displayName = 'ChatRouter';
+
+export default ChatRouter;

package/src/app/[variants]/(main)/chat/_layout/ChatLayout.tsx

@@ -0,0 +1,22 @@
+'use client';
+
+import { PropsWithChildren, memo } from 'react';
+
+import Desktop from './Desktop';
+import Mobile from './Mobile';
+
+interface ChatLayoutProps extends PropsWithChildren {
+  mobile?: boolean;
+}
+
+const ChatLayout = memo<ChatLayoutProps>(({ children, mobile }) => {
+  if (mobile) {
+    return <Mobile>{children}</Mobile>;
+  }
+
+  return <Desktop>{children}</Desktop>;
+});
+
+ChatLayout.displayName = 'ChatLayout';
+
+export default ChatLayout;