@lobehub/chat 1.66.6 → 1.67.0

package/packages/web-crawler/src/utils/appUrlRules.ts

@@ -0,0 +1,40 @@
+import { CrawlUrlRule } from '../type';
+
+export const applyUrlRules = (
+  url: string,
+  urlRules: CrawlUrlRule[],
+): {
+  filterOptions?: CrawlUrlRule['filterOptions'];
+  transformedUrl: string;
+} => {
+  for (const rule of urlRules) {
+    // 转换为正则表达式
+    const regex = new RegExp(rule.urlPattern);
+    const match = url.match(regex);
+
+    if (match) {
+      if (rule.urlTransform) {
+        // 如果有转换规则，进行 URL 转换
+        // 替换 $1, $2 等占位符为捕获组内容
+        const transformedUrl = rule.urlTransform.replaceAll(
+          /\$(\d+)/g,
+          (_, index) => match[parseInt(index)] || '',
+        );
+
+        return {
+          filterOptions: rule.filterOptions,
+          transformedUrl,
+        };
+      } else {
+        // 没有转换规则但匹配了模式，只返回过滤选项
+        return {
+          filterOptions: rule.filterOptions,
+          transformedUrl: url,
+        };
+      }
+    }
+  }
+
+  // 没有匹配任何规则，返回原始 URL
+  return { transformedUrl: url };
+};

package/packages/web-crawler/src/utils/errorType.ts

@@ -0,0 +1,12 @@
+export class PageNotFoundError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'PageNotFoundError';
+  }
+}
+export class NetworkConnectionError extends Error {
+  constructor() {
+    super('Network connection error');
+    this.name = 'NetworkConnectionError';
+  }
+}