@lobehub/chat 1.81.3 → 1.81.5

package/packages/file-loaders/src/loadFile.ts

@@ -0,0 +1,206 @@
+import { stat } from 'node:fs/promises';
+import * as path from 'node:path';
+
+import { fileLoaders } from './loaders';
+import { TextLoader } from './loaders/text';
+import { FileDocument, FileMetadata, SupportedFileType } from './types';
+import type { DocumentPage, FileLoaderInterface } from './types';
+import { isTextReadableFile } from './utils/isTextReadableFile';
+
+/**
+ * Determines the file type based on the filename extension.
+ * @param filePath The path to the file.
+ * @returns The determined file type or 'txt' if text-readable, undefined otherwise.
+ */
+const getFileType = (filePath: string): SupportedFileType | undefined => {
+  const extension = path.extname(filePath).toLowerCase().replace('.', '');
+
+  if (!extension) return 'txt'; // Treat files without extension as text?
+
+  // Prioritize checking if it's a generally text-readable type
+  if (isTextReadableFile(extension)) {
+    return 'txt';
+  }
+
+  // Handle specific non-text or complex types
+  switch (extension) {
+    case 'pdf': {
+      return 'pdf';
+    }
+    case 'docx': {
+      return 'docx';
+    }
+    case 'xlsx':
+    case 'xls': {
+      return 'excel';
+    }
+    case 'pptx': {
+      return 'pptx';
+    }
+    default: {
+      // If not text-readable and not a specific known type, it's unsupported
+      return undefined;
+    }
+  }
+};
+
+// Default fallback loader class
+const DefaultLoader = TextLoader;
+
+/**
+ * Loads a file from the specified path, automatically detecting the file type
+ * and using the appropriate loader class.
+ *
+ * @param filePath The path to the file to load.
+ * @param fileMetadata Optional metadata to override information read from the filesystem.
+ * @returns A Promise resolving to a FileDocument object.
+ */
+export const loadFile = async (
+  filePath: string,
+  fileMetadata?: FileMetadata,
+): Promise<FileDocument> => {
+  let stats;
+  let fsError: string | undefined;
+
+  try {
+    stats = await stat(filePath);
+  } catch (e) {
+    const error = e as Error;
+    console.error(`Error getting file stats for ${filePath}: ${error.message}`);
+    fsError = `Failed to access file stats: ${error.message}`;
+  }
+
+  // Determine base file info from path and stats (if available)
+  const fileExtension = path.extname(filePath).slice(1).toLowerCase();
+  const baseFilename = path.basename(filePath);
+
+  // Apply overrides from fileMetadata or use defaults
+  const source = fileMetadata?.source ?? filePath;
+  const filename = fileMetadata?.filename ?? baseFilename;
+  const fileType = fileMetadata?.fileType ?? fileExtension;
+  const createdTime = fileMetadata?.createdTime ?? stats?.ctime ?? new Date();
+  const modifiedTime = fileMetadata?.modifiedTime ?? stats?.mtime ?? new Date();
+
+  const paserType = getFileType(filePath);
+
+  // Select the loader CLASS based on the determined fileType, fallback to DefaultLoader
+  const LoaderClass: new () => FileLoaderInterface = paserType
+    ? fileLoaders[paserType]
+    : DefaultLoader;
+
+  if (!paserType) {
+    console.warn(
+      `No specific loader found for file type '${fileType}'. Using default loader (${DefaultLoader.name}) as fallback.`,
+    );
+  }
+
+  let pages: DocumentPage[] = [];
+  let aggregatedContent = '';
+  let loaderError: string | undefined;
+  let aggregationError: string | undefined;
+  let metadataError: string | undefined;
+  let loaderSpecificMetadata: any | undefined;
+
+  // Instantiate the loader
+  const loaderInstance = new LoaderClass();
+
+  // If we couldn't even get stats, skip loader execution
+  if (!fsError) {
+    try {
+      // 1. Load pages using the instance
+      pages = await loaderInstance.loadPages(filePath);
+
+      try {
+        // 2. Aggregate content using the instance
+        aggregatedContent = await loaderInstance.aggregateContent(pages);
+      } catch (aggError) {
+        const error = aggError as Error;
+        console.error(
+          `Error aggregating content for ${filePath} using ${LoaderClass.name}: ${error.message}`,
+        );
+        aggregationError = `Content aggregation failed: ${error.message}`;
+        // Keep the pages loaded, but content might be empty/incomplete
+      }
+
+      // 3. Attach document-specific metadata if loader supports it
+      if (typeof loaderInstance.attachDocumentMetadata === 'function') {
+        try {
+          loaderSpecificMetadata = await loaderInstance.attachDocumentMetadata(filePath);
+        } catch (metaErr) {
+          const error = metaErr as Error;
+          console.error(
+            `Error attaching metadata for ${filePath} using ${LoaderClass.name}: ${error.message}`,
+          );
+          metadataError = `Metadata attachment failed: ${error.message}`;
+        }
+      }
+    } catch (loadErr) {
+      const error = loadErr as Error;
+      console.error(
+        `Error loading pages for ${filePath} using ${LoaderClass.name}: ${error.message}`,
+      );
+      loaderError = `Loader execution failed: ${error.message}`;
+      // Provide a minimal error page if loader failed critically
+      pages = [
+        {
+          charCount: 0,
+          lineCount: 0,
+          metadata: { error: loaderError },
+          pageContent: '',
+        },
+      ];
+      // Aggregated content remains empty
+    }
+  } else {
+    // If stats failed, create a minimal error page
+    pages = [
+      {
+        charCount: 0,
+        lineCount: 0,
+        metadata: { error: fsError },
+        pageContent: '',
+      },
+    ];
+    // Aggregated content remains empty
+  }
+
+  // Calculate totals from the loaded pages
+  let totalCharCount = 0;
+  let totalLineCount = 0;
+  for (const page of pages) {
+    totalCharCount += page.charCount;
+    totalLineCount += page.lineCount;
+  }
+
+  // Combine all potential errors
+  const combinedError =
+    [fsError, loaderError, aggregationError, metadataError].filter(Boolean).join('; ') || undefined;
+
+  // Construct the final FileDocument
+  const fileDocument: FileDocument = {
+    content: aggregatedContent, // Use content from aggregateContent
+    createdTime,
+    fileType,
+    filename,
+    metadata: {
+      // Include combined errors
+      error: combinedError,
+      // Add loader specific metadata under a namespace
+      loaderSpecific: loaderSpecificMetadata ?? undefined,
+      // Add other file-level metadata
+      ...fileMetadata,
+    },
+    modifiedTime,
+    pages, // Use pages from loadPages
+    source,
+    totalCharCount,
+    totalLineCount,
+  };
+
+  // Clean up undefined error field if no error occurred
+  if (!fileDocument.metadata.error) {
+    delete fileDocument.metadata.error;
+  }
+
+  return fileDocument;
+};

package/packages/file-loaders/src/loaders/docx/__snapshots__/index.test.ts.snap

@@ -0,0 +1,74 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`DocxLoader > should aggregate content correctly > aggregated_content 1`] = `
+"简单报告
+
+副标题
+
+轻点或点按此占位符文本并开始键入即可开始。你可以在 Mac、iPad、iPhone 或 iCloud.com 上查看和编辑此文稿。
+
+轻松编辑文本、更改字体以及添加精美的图形。使用段落样式来使整篇文稿保持一致的风格。例如，此段落使用“正文”样式。你可以在“格式”控制的“文本”标签页中更改样式。
+
+若要添加照片、图像画廊、音频片段、视频、图表或任意 700 多种可自定义形状，请在工具栏中轻点或点按其中一个插入按钮，或者将对象拖放到页面中。你可以分层放置对象、调整其大小以及将其放在页面中的任意位置。若要更改对象随文本移动的方式，请选择对象并随后轻点或点按“格式”控制中的“排列”标签页。
+
+小标题
+
+Pages 文稿可用于文字处理和页面布局。此“简单报告”模板为文字处理而设置，如此一来，文本便会随着你的键入而从某一页流向下一页，到达页面末尾时会自动创建新的页面。
+
+在页面布局文稿中，你可以手动重新排列页面并随意调整页面中的文本框、图像和其他对象的位置。若要创建页面布局文稿，请在模板选取器中选取一种页面布局模板。你也可以在 Mac、iPad 或 iPhone 上将此文稿改为页面布局，方法是在“文稿”控制中关闭“文稿正文”。
+
+“这是一个引用（报告中的关键短语）的例子。轻点或点按此文本添加你自己的内容。”
+
+
+
+
+
+
+
+
+
+这是第二页的内容
+
+"
+`;
+
+exports[`DocxLoader > should load pages correctly from a DOCX file 1`] = `
+[
+  {
+    "charCount": 587,
+    "lineCount": 29,
+    "metadata": {
+      "pageNumber": 1,
+    },
+    "pageContent": "简单报告
+
+副标题
+
+轻点或点按此占位符文本并开始键入即可开始。你可以在 Mac、iPad、iPhone 或 iCloud.com 上查看和编辑此文稿。
+
+轻松编辑文本、更改字体以及添加精美的图形。使用段落样式来使整篇文稿保持一致的风格。例如，此段落使用“正文”样式。你可以在“格式”控制的“文本”标签页中更改样式。
+
+若要添加照片、图像画廊、音频片段、视频、图表或任意 700 多种可自定义形状，请在工具栏中轻点或点按其中一个插入按钮，或者将对象拖放到页面中。你可以分层放置对象、调整其大小以及将其放在页面中的任意位置。若要更改对象随文本移动的方式，请选择对象并随后轻点或点按“格式”控制中的“排列”标签页。
+
+小标题
+
+Pages 文稿可用于文字处理和页面布局。此“简单报告”模板为文字处理而设置，如此一来，文本便会随着你的键入而从某一页流向下一页，到达页面末尾时会自动创建新的页面。
+
+在页面布局文稿中，你可以手动重新排列页面并随意调整页面中的文本框、图像和其他对象的位置。若要创建页面布局文稿，请在模板选取器中选取一种页面布局模板。你也可以在 Mac、iPad 或 iPhone 上将此文稿改为页面布局，方法是在“文稿”控制中关闭“文稿正文”。
+
+“这是一个引用（报告中的关键短语）的例子。轻点或点按此文本添加你自己的内容。”
+
+
+
+
+
+
+
+
+
+这是第二页的内容
+
+",
+  },
+]
+`;

package/packages/file-loaders/src/loaders/docx/index.test.ts

@@ -0,0 +1,41 @@
+import path from 'node:path';
+import { beforeEach, describe, expect, it } from 'vitest';
+
+import type { FileLoaderInterface } from '../../types';
+import { DocxLoader } from './index';
+
+// 确保你已经在 fixtures 目录下放置了 test.docx 文件
+const fixturePath = (filename: string) => path.join(__dirname, `./fixtures/${filename}`);
+
+let loader: FileLoaderInterface;
+
+const testFile = fixturePath('test.docx');
+const nonExistentFile = fixturePath('nonexistent.docx');
+
+beforeEach(() => {
+  loader = new DocxLoader();
+});
+
+describe('DocxLoader', () => {
+  it('should load pages correctly from a DOCX file', async () => {
+    const pages = await loader.loadPages(testFile);
+    // DOCX 通常加载为单个页面
+    expect(pages).toHaveLength(1);
+    expect(pages).toMatchSnapshot();
+  });
+
+  it('should aggregate content correctly', async () => {
+    const pages = await loader.loadPages(testFile);
+    const content = await loader.aggregateContent(pages);
+    // 对于单页文档，聚合内容应与页面内容相同
+    expect(content).toEqual(pages[0].pageContent);
+    expect(content).toMatchSnapshot('aggregated_content');
+  });
+
+  it('should handle file read errors in loadPages', async () => {
+    const pages = await loader.loadPages(nonExistentFile);
+    expect(pages).toHaveLength(1); // 即使失败也返回一个包含错误信息的页面
+    expect(pages[0].pageContent).toBe('');
+    expect(pages[0].metadata.error).toContain('Failed to load DOCX file');
+  });
+});

package/packages/file-loaders/src/loaders/docx/index.ts

@@ -0,0 +1,73 @@
+import { DocxLoader as LangchainDocxLoader } from '@langchain/community/document_loaders/fs/docx';
+
+import type { DocumentPage, FileLoaderInterface } from '../../types';
+
+/**
+ * Loads Word documents (.docx) using the LangChain Community DocxLoader.
+ */
+export class DocxLoader implements FileLoaderInterface {
+  async loadPages(filePath: string): Promise<DocumentPage[]> {
+    try {
+      const loader = new LangchainDocxLoader(filePath);
+      const docs = await loader.load(); // Langchain DocxLoader typically loads the whole doc as one
+
+      const pages: DocumentPage[] = docs.map((doc) => {
+        const pageContent = doc.pageContent || '';
+        const lines = pageContent.split('\n');
+        const lineCount = lines.length;
+        const charCount = pageContent.length;
+
+        // Langchain DocxLoader doesn't usually provide page numbers in metadata
+        // We treat it as a single page
+        const metadata = {
+          ...doc.metadata, // Include any other metadata Langchain provides
+          pageNumber: 1,
+        };
+
+        // @ts-expect-error Remove source if present, as it's handled at the FileDocument level
+        delete metadata.source;
+
+        return {
+          charCount,
+          lineCount,
+          metadata,
+          pageContent,
+        };
+      });
+
+      // If docs array is empty (e.g., empty file), create an empty page
+      if (pages.length === 0) {
+        pages.push({
+          charCount: 0,
+          lineCount: 0,
+          metadata: { pageNumber: 1 },
+          pageContent: '',
+        });
+      }
+
+      return pages;
+    } catch (e) {
+      const error = e as Error;
+      console.error(`Error loading DOCX file ${filePath} using LangChain loader: ${error.message}`);
+      const errorPage: DocumentPage = {
+        charCount: 0,
+        lineCount: 0,
+        metadata: {
+          error: `Failed to load DOCX file: ${error.message}`,
+        },
+        pageContent: '',
+      };
+      return [errorPage];
+    }
+  }
+
+  /**
+   * Aggregates content from DOCX pages.
+   * Uses double newline as a separator.
+   * @param pages Array of DocumentPage objects.
+   * @returns Aggregated content as a string.
+   */
+  async aggregateContent(pages: DocumentPage[]): Promise<string> {
+    return pages.map((page) => page.pageContent).join('\n\n');
+  }
+}

package/packages/file-loaders/src/loaders/excel/__snapshots__/index.test.ts.snap

@@ -0,0 +1,58 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`ExcelLoader > should aggregate content correctly (joining sheets) > aggregated_content 1`] = `
+"## Sheet: 表1
+
+| __EMPTY | 类别 A | 类别 B | __EMPTY_1 | __EMPTY_2 |
+| --- | --- | --- | --- | --- |
+| 项目 1 | 5 | 7 |  |  |
+| 项目 2 | 10 | 8 |  |  |
+| 项目 3 | 9 | 15 |  |  |
+| 项目 4 | 7 | 12 |  |  |
+| 项目 5 | 16 | 21 |  |  |
+
+---
+
+## Sheet: 表2 - 表格 2
+
+| __EMPTY | 类别 A | 类别 B | __EMPTY_1 | __EMPTY_2 |
+| --- | --- | --- | --- | --- |
+| 项目 1 | 5 | 7 |  |  |
+| 项目 2 | 10 | 8 |  |  |
+| 项目 3 | 9 | 15 |  |  |
+| 项目 4 | 7 | 12 |  |  |
+| 项目 5 | 16 | 21 |  |  |"
+`;
+
+exports[`ExcelLoader > should load pages correctly from an Excel file (one page per sheet) 1`] = `
+[
+  {
+    "charCount": 201,
+    "lineCount": 7,
+    "metadata": {
+      "sheetName": "表1",
+    },
+    "pageContent": "| __EMPTY | 类别 A | 类别 B | __EMPTY_1 | __EMPTY_2 |
+| --- | --- | --- | --- | --- |
+| 项目 1 | 5 | 7 |  |  |
+| 项目 2 | 10 | 8 |  |  |
+| 项目 3 | 9 | 15 |  |  |
+| 项目 4 | 7 | 12 |  |  |
+| 项目 5 | 16 | 21 |  |  |",
+  },
+  {
+    "charCount": 201,
+    "lineCount": 7,
+    "metadata": {
+      "sheetName": "表2 - 表格 2",
+    },
+    "pageContent": "| __EMPTY | 类别 A | 类别 B | __EMPTY_1 | __EMPTY_2 |
+| --- | --- | --- | --- | --- |
+| 项目 1 | 5 | 7 |  |  |
+| 项目 2 | 10 | 8 |  |  |
+| 项目 3 | 9 | 15 |  |  |
+| 项目 4 | 7 | 12 |  |  |
+| 项目 5 | 16 | 21 |  |  |",
+  },
+]
+`;

package/packages/file-loaders/src/loaders/excel/index.test.ts

@@ -0,0 +1,47 @@
+import path from 'node:path';
+import { beforeEach, describe, expect, it } from 'vitest';
+
+import type { FileLoaderInterface } from '../../types';
+import { ExcelLoader } from './index';
+
+// 确保你已经在 fixtures 目录下放置了 test.xlsx 文件
+// 这个 Excel 文件最好包含多个工作表 (sheets) 以便测试
+const fixturePath = (filename: string) => path.join(__dirname, `./fixtures/${filename}`);
+
+let loader: FileLoaderInterface;
+
+const testFile = fixturePath('test.xlsx');
+const nonExistentFile = fixturePath('nonexistent.xlsx');
+
+beforeEach(() => {
+  loader = new ExcelLoader();
+});
+
+describe('ExcelLoader', () => {
+  it('should load pages correctly from an Excel file (one page per sheet)', async () => {
+    const pages = await loader.loadPages(testFile);
+    // Excel 文件有多少个 sheet，就应该有多少个 page
+    expect(pages.length).toBeGreaterThan(0);
+
+    // 直接对整个 pages 数组进行快照测试
+    expect(pages).toMatchSnapshot();
+
+    // 如果你的 test.xlsx 有多个 sheet，可以添加更多断言
+    // 例如检查特定 sheet 的 metadata 中的 sheetName
+    // expect(pages[1].metadata.sheetName).toBe('Sheet2');
+  });
+
+  it('should aggregate content correctly (joining sheets)', async () => {
+    const pages = await loader.loadPages(testFile);
+    const content = await loader.aggregateContent(pages);
+    // 默认聚合是以换行符连接各 sheet 内容
+    expect(content).toMatchSnapshot('aggregated_content');
+  });
+
+  it('should handle file read errors in loadPages', async () => {
+    const pages = await loader.loadPages(nonExistentFile);
+    expect(pages).toHaveLength(1); // 即使失败也返回一个包含错误信息的页面
+    expect(pages[0].pageContent).toBe('');
+    expect(pages[0].metadata.error).toContain('Failed to load Excel file');
+  });
+});

package/packages/file-loaders/src/loaders/excel/index.ts

@@ -0,0 +1,121 @@
+import { readFile } from 'node:fs/promises';
+import * as xlsx from 'xlsx';
+
+import type { DocumentPage, FileLoaderInterface } from '../../types';
+
+/**
+ * Converts sheet data (array of objects) to a Markdown table string.
+ * Handles empty sheets and escapes pipe characters.
+ */
+function sheetToMarkdownTable(jsonData: Record<string, any>[]): string {
+  if (!jsonData || jsonData.length === 0) {
+    return '*Sheet is empty or contains no data.*';
+  }
+
+  // Ensure all rows have the same keys based on the first row, handle potentially sparse data
+  const headers = Object.keys(jsonData[0] || {});
+  if (headers.length === 0) {
+    return '*Sheet has headers but no data.*';
+  }
+
+  const headerRow = `| ${headers.join(' | ')} |`;
+  const separatorRow = `| ${headers.map(() => '---').join(' | ')} |`;
+
+  const dataRows = jsonData
+    .map((row) => {
+      const cells = headers.map((header) => {
+        const value = row[header];
+        // Handle null/undefined and escape pipe characters within cells
+        const cellContent =
+          value === null || value === undefined ? '' : String(value).replaceAll('|', '\\|');
+        return cellContent.trim(); // Trim whitespace from cells
+      });
+      return `| ${cells.join(' | ')} |`;
+    })
+    .join('\n');
+
+  return `${headerRow}\n${separatorRow}\n${dataRows}`;
+}
+
+/**
+ * Loads Excel files (.xlsx, .xls) using the 'xlsx' library.
+ * Each sheet becomes a DocumentPage containing a Markdown table generated by sheetToMarkdownTable.
+ */
+export class ExcelLoader implements FileLoaderInterface {
+  async loadPages(filePath: string): Promise<DocumentPage[]> {
+    const pages: DocumentPage[] = [];
+    try {
+      // Use readFile for async operation compatible with other loaders
+      const dataBuffer = await readFile(filePath);
+      const workbook = xlsx.read(dataBuffer, { type: 'buffer' });
+
+      for (const sheetName of workbook.SheetNames) {
+        const worksheet = workbook.Sheets[sheetName];
+        // Use sheet_to_json to get array of objects for our custom markdown function
+        const jsonData = xlsx.utils.sheet_to_json<Record<string, any>>(worksheet, {
+          // Get formatted strings, not raw values
+          defval: '',
+          raw: false, // Use empty string for blank cells
+        });
+
+        // Convert to markdown using YOUR helper function
+        const tableMarkdown = sheetToMarkdownTable(jsonData);
+
+        const lines = tableMarkdown.split('\n');
+        const lineCount = lines.length;
+        const charCount = tableMarkdown.length;
+
+        pages.push({
+          // Trim whitespace
+          charCount,
+          lineCount,
+          metadata: {
+            sheetName: sheetName,
+          },
+          pageContent: tableMarkdown.trim(),
+        });
+      }
+
+      if (pages.length === 0) {
+        pages.push({
+          charCount: 0,
+          lineCount: 0,
+          metadata: {
+            error: 'Excel file contains no sheets.',
+          },
+          pageContent: '',
+        });
+      }
+
+      return pages;
+    } catch (e) {
+      const error = e as Error;
+      console.error(`Error loading Excel file ${filePath}: ${error.message}`);
+      const errorPage: DocumentPage = {
+        charCount: 0,
+        lineCount: 0,
+        metadata: {
+          error: `Failed to load Excel file: ${error.message}`,
+        },
+        pageContent: '',
+      };
+      return [errorPage];
+    }
+  }
+
+  /**
+   * Aggregates content from Excel sheets (Markdown tables).
+   * Adds the sheet name as a header before each table.
+   * @param pages Array of DocumentPage objects from loadPages.
+   * @returns Aggregated content as a string.
+   */
+  async aggregateContent(pages: DocumentPage[]): Promise<string> {
+    return pages
+      .map((page) => {
+        const sheetName = page.metadata.sheetName;
+        const header = sheetName ? `## Sheet: ${sheetName}\n\n` : '';
+        return header + page.pageContent;
+      })
+      .join('\n\n---\n\n'); // Separator between sheets
+  }
+}

package/packages/file-loaders/src/loaders/index.ts

@@ -0,0 +1,19 @@
+import { FileLoaderInterface, SupportedFileType } from '../types';
+import { DocxLoader } from './docx';
+// import { EpubLoader } from './epub';
+import { ExcelLoader } from './excel';
+import { PdfLoader } from './pdf';
+import { PptxLoader } from './pptx';
+import { TextLoader } from './text';
+
+// Loader configuration map
+// Key: file extension (lowercase, without leading dot) or specific type name
+// Value: Loader Class implementing FileLoaderInterface
+export const fileLoaders: Record<SupportedFileType, new () => FileLoaderInterface> = {
+  docx: DocxLoader,
+  // epub: EpubLoader,
+  excel: ExcelLoader,
+  pdf: PdfLoader,
+  pptx: PptxLoader,
+  txt: TextLoader,
+};