@lobehub/chat 1.81.4 → 1.81.6

package/packages/file-loaders/src/loaders/docx/index.ts

@@ -0,0 +1,73 @@
+import { DocxLoader as LangchainDocxLoader } from '@langchain/community/document_loaders/fs/docx';
+
+import type { DocumentPage, FileLoaderInterface } from '../../types';
+
+/**
+ * Loads Word documents (.docx) using the LangChain Community DocxLoader.
+ */
+export class DocxLoader implements FileLoaderInterface {
+  async loadPages(filePath: string): Promise<DocumentPage[]> {
+    try {
+      const loader = new LangchainDocxLoader(filePath);
+      const docs = await loader.load(); // Langchain DocxLoader typically loads the whole doc as one
+
+      const pages: DocumentPage[] = docs.map((doc) => {
+        const pageContent = doc.pageContent || '';
+        const lines = pageContent.split('\n');
+        const lineCount = lines.length;
+        const charCount = pageContent.length;
+
+        // Langchain DocxLoader doesn't usually provide page numbers in metadata
+        // We treat it as a single page
+        const metadata = {
+          ...doc.metadata, // Include any other metadata Langchain provides
+          pageNumber: 1,
+        };
+
+        // @ts-expect-error Remove source if present, as it's handled at the FileDocument level
+        delete metadata.source;
+
+        return {
+          charCount,
+          lineCount,
+          metadata,
+          pageContent,
+        };
+      });
+
+      // If docs array is empty (e.g., empty file), create an empty page
+      if (pages.length === 0) {
+        pages.push({
+          charCount: 0,
+          lineCount: 0,
+          metadata: { pageNumber: 1 },
+          pageContent: '',
+        });
+      }
+
+      return pages;
+    } catch (e) {
+      const error = e as Error;
+      console.error(`Error loading DOCX file ${filePath} using LangChain loader: ${error.message}`);
+      const errorPage: DocumentPage = {
+        charCount: 0,
+        lineCount: 0,
+        metadata: {
+          error: `Failed to load DOCX file: ${error.message}`,
+        },
+        pageContent: '',
+      };
+      return [errorPage];
+    }
+  }
+
+  /**
+   * Aggregates content from DOCX pages.
+   * Uses double newline as a separator.
+   * @param pages Array of DocumentPage objects.
+   * @returns Aggregated content as a string.
+   */
+  async aggregateContent(pages: DocumentPage[]): Promise<string> {
+    return pages.map((page) => page.pageContent).join('\n\n');
+  }
+}

package/packages/file-loaders/src/loaders/excel/__snapshots__/index.test.ts.snap

@@ -0,0 +1,58 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`ExcelLoader > should aggregate content correctly (joining sheets) > aggregated_content 1`] = `
+"## Sheet: 表1
+
+| __EMPTY | 类别 A | 类别 B | __EMPTY_1 | __EMPTY_2 |
+| --- | --- | --- | --- | --- |
+| 项目 1 | 5 | 7 |  |  |
+| 项目 2 | 10 | 8 |  |  |
+| 项目 3 | 9 | 15 |  |  |
+| 项目 4 | 7 | 12 |  |  |
+| 项目 5 | 16 | 21 |  |  |
+
+---
+
+## Sheet: 表2 - 表格 2
+
+| __EMPTY | 类别 A | 类别 B | __EMPTY_1 | __EMPTY_2 |
+| --- | --- | --- | --- | --- |
+| 项目 1 | 5 | 7 |  |  |
+| 项目 2 | 10 | 8 |  |  |
+| 项目 3 | 9 | 15 |  |  |
+| 项目 4 | 7 | 12 |  |  |
+| 项目 5 | 16 | 21 |  |  |"
+`;
+
+exports[`ExcelLoader > should load pages correctly from an Excel file (one page per sheet) 1`] = `
+[
+  {
+    "charCount": 201,
+    "lineCount": 7,
+    "metadata": {
+      "sheetName": "表1",
+    },
+    "pageContent": "| __EMPTY | 类别 A | 类别 B | __EMPTY_1 | __EMPTY_2 |
+| --- | --- | --- | --- | --- |
+| 项目 1 | 5 | 7 |  |  |
+| 项目 2 | 10 | 8 |  |  |
+| 项目 3 | 9 | 15 |  |  |
+| 项目 4 | 7 | 12 |  |  |
+| 项目 5 | 16 | 21 |  |  |",
+  },
+  {
+    "charCount": 201,
+    "lineCount": 7,
+    "metadata": {
+      "sheetName": "表2 - 表格 2",
+    },
+    "pageContent": "| __EMPTY | 类别 A | 类别 B | __EMPTY_1 | __EMPTY_2 |
+| --- | --- | --- | --- | --- |
+| 项目 1 | 5 | 7 |  |  |
+| 项目 2 | 10 | 8 |  |  |
+| 项目 3 | 9 | 15 |  |  |
+| 项目 4 | 7 | 12 |  |  |
+| 项目 5 | 16 | 21 |  |  |",
+  },
+]
+`;

package/packages/file-loaders/src/loaders/excel/index.test.ts

@@ -0,0 +1,47 @@
+import path from 'node:path';
+import { beforeEach, describe, expect, it } from 'vitest';
+
+import type { FileLoaderInterface } from '../../types';
+import { ExcelLoader } from './index';
+
+// 确保你已经在 fixtures 目录下放置了 test.xlsx 文件
+// 这个 Excel 文件最好包含多个工作表 (sheets) 以便测试
+const fixturePath = (filename: string) => path.join(__dirname, `./fixtures/${filename}`);
+
+let loader: FileLoaderInterface;
+
+const testFile = fixturePath('test.xlsx');
+const nonExistentFile = fixturePath('nonexistent.xlsx');
+
+beforeEach(() => {
+  loader = new ExcelLoader();
+});
+
+describe('ExcelLoader', () => {
+  it('should load pages correctly from an Excel file (one page per sheet)', async () => {
+    const pages = await loader.loadPages(testFile);
+    // Excel 文件有多少个 sheet，就应该有多少个 page
+    expect(pages.length).toBeGreaterThan(0);
+
+    // 直接对整个 pages 数组进行快照测试
+    expect(pages).toMatchSnapshot();
+
+    // 如果你的 test.xlsx 有多个 sheet，可以添加更多断言
+    // 例如检查特定 sheet 的 metadata 中的 sheetName
+    // expect(pages[1].metadata.sheetName).toBe('Sheet2');
+  });
+
+  it('should aggregate content correctly (joining sheets)', async () => {
+    const pages = await loader.loadPages(testFile);
+    const content = await loader.aggregateContent(pages);
+    // 默认聚合是以换行符连接各 sheet 内容
+    expect(content).toMatchSnapshot('aggregated_content');
+  });
+
+  it('should handle file read errors in loadPages', async () => {
+    const pages = await loader.loadPages(nonExistentFile);
+    expect(pages).toHaveLength(1); // 即使失败也返回一个包含错误信息的页面
+    expect(pages[0].pageContent).toBe('');
+    expect(pages[0].metadata.error).toContain('Failed to load Excel file');
+  });
+});

package/packages/file-loaders/src/loaders/excel/index.ts

@@ -0,0 +1,121 @@
+import { readFile } from 'node:fs/promises';
+import * as xlsx from 'xlsx';
+
+import type { DocumentPage, FileLoaderInterface } from '../../types';
+
+/**
+ * Converts sheet data (array of objects) to a Markdown table string.
+ * Handles empty sheets and escapes pipe characters.
+ */
+function sheetToMarkdownTable(jsonData: Record<string, any>[]): string {
+  if (!jsonData || jsonData.length === 0) {
+    return '*Sheet is empty or contains no data.*';
+  }
+
+  // Ensure all rows have the same keys based on the first row, handle potentially sparse data
+  const headers = Object.keys(jsonData[0] || {});
+  if (headers.length === 0) {
+    return '*Sheet has headers but no data.*';
+  }
+
+  const headerRow = `| ${headers.join(' | ')} |`;
+  const separatorRow = `| ${headers.map(() => '---').join(' | ')} |`;
+
+  const dataRows = jsonData
+    .map((row) => {
+      const cells = headers.map((header) => {
+        const value = row[header];
+        // Handle null/undefined and escape pipe characters within cells
+        const cellContent =
+          value === null || value === undefined ? '' : String(value).replaceAll('|', '\\|');
+        return cellContent.trim(); // Trim whitespace from cells
+      });
+      return `| ${cells.join(' | ')} |`;
+    })
+    .join('\n');
+
+  return `${headerRow}\n${separatorRow}\n${dataRows}`;
+}
+
+/**
+ * Loads Excel files (.xlsx, .xls) using the 'xlsx' library.
+ * Each sheet becomes a DocumentPage containing a Markdown table generated by sheetToMarkdownTable.
+ */
+export class ExcelLoader implements FileLoaderInterface {
+  async loadPages(filePath: string): Promise<DocumentPage[]> {
+    const pages: DocumentPage[] = [];
+    try {
+      // Use readFile for async operation compatible with other loaders
+      const dataBuffer = await readFile(filePath);
+      const workbook = xlsx.read(dataBuffer, { type: 'buffer' });
+
+      for (const sheetName of workbook.SheetNames) {
+        const worksheet = workbook.Sheets[sheetName];
+        // Use sheet_to_json to get array of objects for our custom markdown function
+        const jsonData = xlsx.utils.sheet_to_json<Record<string, any>>(worksheet, {
+          // Get formatted strings, not raw values
+          defval: '',
+          raw: false, // Use empty string for blank cells
+        });
+
+        // Convert to markdown using YOUR helper function
+        const tableMarkdown = sheetToMarkdownTable(jsonData);
+
+        const lines = tableMarkdown.split('\n');
+        const lineCount = lines.length;
+        const charCount = tableMarkdown.length;
+
+        pages.push({
+          // Trim whitespace
+          charCount,
+          lineCount,
+          metadata: {
+            sheetName: sheetName,
+          },
+          pageContent: tableMarkdown.trim(),
+        });
+      }
+
+      if (pages.length === 0) {
+        pages.push({
+          charCount: 0,
+          lineCount: 0,
+          metadata: {
+            error: 'Excel file contains no sheets.',
+          },
+          pageContent: '',
+        });
+      }
+
+      return pages;
+    } catch (e) {
+      const error = e as Error;
+      console.error(`Error loading Excel file ${filePath}: ${error.message}`);
+      const errorPage: DocumentPage = {
+        charCount: 0,
+        lineCount: 0,
+        metadata: {
+          error: `Failed to load Excel file: ${error.message}`,
+        },
+        pageContent: '',
+      };
+      return [errorPage];
+    }
+  }
+
+  /**
+   * Aggregates content from Excel sheets (Markdown tables).
+   * Adds the sheet name as a header before each table.
+   * @param pages Array of DocumentPage objects from loadPages.
+   * @returns Aggregated content as a string.
+   */
+  async aggregateContent(pages: DocumentPage[]): Promise<string> {
+    return pages
+      .map((page) => {
+        const sheetName = page.metadata.sheetName;
+        const header = sheetName ? `## Sheet: ${sheetName}\n\n` : '';
+        return header + page.pageContent;
+      })
+      .join('\n\n---\n\n'); // Separator between sheets
+  }
+}

package/packages/file-loaders/src/loaders/index.ts

@@ -0,0 +1,19 @@
+import { FileLoaderInterface, SupportedFileType } from '../types';
+import { DocxLoader } from './docx';
+// import { EpubLoader } from './epub';
+import { ExcelLoader } from './excel';
+import { PdfLoader } from './pdf';
+import { PptxLoader } from './pptx';
+import { TextLoader } from './text';
+
+// Loader configuration map
+// Key: file extension (lowercase, without leading dot) or specific type name
+// Value: Loader Class implementing FileLoaderInterface
+export const fileLoaders: Record<SupportedFileType, new () => FileLoaderInterface> = {
+  docx: DocxLoader,
+  // epub: EpubLoader,
+  excel: ExcelLoader,
+  pdf: PdfLoader,
+  pptx: PptxLoader,
+  txt: TextLoader,
+};

package/packages/file-loaders/src/loaders/pdf/__snapshots__/index.test.ts.snap

@@ -0,0 +1,98 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`PdfLoader > should aggregate content correctly 1`] = `
+"简单报告
+副标题
+轻点或点按此占位符⽂本并开始键⼊即可开始。你可以在 Mac、iPad、iPhone 或
+iCloud.com 上查看和编辑此⽂稿。
+轻松编辑⽂本、更改字体以及添加精美的图形。使⽤段落样式来使整篇⽂稿保持⼀
+致的⻛格。例如，此段落使⽤“正⽂”样式。你可以在“格式”控制的“⽂本”标签⻚中
+更改样式。
+若要添加照⽚、图像画廊、⾳频⽚段、视频、图表或任意 700 多种可⾃定义形状，
+请在⼯具栏中轻点或点按其中⼀个插⼊按钮，或者将对象拖放到⻚⾯中。你可以分
+层放置对象、调整其⼤⼩以及将其放在⻚⾯中的任意位置。若要更改对象随⽂本移
+动的⽅式，请选择对象并随后轻点或点按“格式”控制中的“排列”标签⻚。
+⼩标题
+Pages ⽂稿可⽤于⽂字处理和⻚⾯布局。此“简单报告”模板为⽂字处理⽽设置，如
+此⼀来，⽂本便会随着你的键⼊⽽从某⼀⻚流向下⼀⻚，到达⻚⾯末尾时会⾃动创
+建新的⻚⾯。
+在⻚⾯布局⽂稿中，你可以⼿动重新排列⻚⾯并随意调整⻚⾯中的⽂本框、图像和
+其他对象的位置。若要创建⻚⾯布局⽂稿，请在模板选取器中选取⼀种⻚⾯布局模
+板。你也可以在 Mac、iPad 或 iPhone 上将此⽂稿改为⻚⾯布局，⽅法是在“⽂稿”
+控制中关闭“⽂稿正⽂”。
+“这是⼀个引⽤（报告中的关键短语）的例⼦。轻点或点按此
+⽂本添加你⾃⼰的内容。”
+⻚脚 
+1
+
+这是第⼆⻚的内容
+⻚脚 
+2"
+`;
+
+exports[`PdfLoader > should attach document metadata correctly 1`] = `
+{
+  "pdfInfo": {
+    "CreationDate": "D:20250419155028Z00'00'",
+    "Creator": "Pages文稿",
+    "EncryptFilterName": null,
+    "IsAcroFormPresent": false,
+    "IsCollectionPresent": false,
+    "IsLinearized": false,
+    "IsSignaturesPresent": false,
+    "IsXFAPresent": false,
+    "Language": null,
+    "ModDate": "D:20250419155028Z00'00'",
+    "PDFFormatVersion": "1.3",
+    "Producer": "macOS 版本15.3.2（版号24D81） Quartz PDFContext",
+    "Title": "test",
+  },
+  "pdfMetadata": null,
+  "pdfVersion": "4.8.69",
+}
+`;
+
+exports[`PdfLoader > should load pages correctly from a PDF file 1`] = `
+[
+  {
+    "charCount": 576,
+    "lineCount": 23,
+    "metadata": {
+      "pageNumber": 1,
+    },
+    "pageContent": "简单报告
+副标题
+轻点或点按此占位符⽂本并开始键⼊即可开始。你可以在 Mac、iPad、iPhone 或
+iCloud.com 上查看和编辑此⽂稿。
+轻松编辑⽂本、更改字体以及添加精美的图形。使⽤段落样式来使整篇⽂稿保持⼀
+致的⻛格。例如，此段落使⽤“正⽂”样式。你可以在“格式”控制的“⽂本”标签⻚中
+更改样式。
+若要添加照⽚、图像画廊、⾳频⽚段、视频、图表或任意 700 多种可⾃定义形状，
+请在⼯具栏中轻点或点按其中⼀个插⼊按钮，或者将对象拖放到⻚⾯中。你可以分
+层放置对象、调整其⼤⼩以及将其放在⻚⾯中的任意位置。若要更改对象随⽂本移
+动的⽅式，请选择对象并随后轻点或点按“格式”控制中的“排列”标签⻚。
+⼩标题
+Pages ⽂稿可⽤于⽂字处理和⻚⾯布局。此“简单报告”模板为⽂字处理⽽设置，如
+此⼀来，⽂本便会随着你的键⼊⽽从某⼀⻚流向下⼀⻚，到达⻚⾯末尾时会⾃动创
+建新的⻚⾯。
+在⻚⾯布局⽂稿中，你可以⼿动重新排列⻚⾯并随意调整⻚⾯中的⽂本框、图像和
+其他对象的位置。若要创建⻚⾯布局⽂稿，请在模板选取器中选取⼀种⻚⾯布局模
+板。你也可以在 Mac、iPad 或 iPhone 上将此⽂稿改为⻚⾯布局，⽅法是在“⽂稿”
+控制中关闭“⽂稿正⽂”。
+“这是⼀个引⽤（报告中的关键短语）的例⼦。轻点或点按此
+⽂本添加你⾃⼰的内容。”
+⻚脚 
+1",
+  },
+  {
+    "charCount": 14,
+    "lineCount": 3,
+    "metadata": {
+      "pageNumber": 2,
+    },
+    "pageContent": "这是第⼆⻚的内容
+⻚脚 
+2",
+  },
+]
+`;

package/packages/file-loaders/src/loaders/pdf/index.test.ts

@@ -0,0 +1,49 @@
+// @vitest-environment node
+import path from 'node:path';
+import { beforeEach, describe, expect, it } from 'vitest';
+
+import type { FileLoaderInterface } from '../../types';
+import { PdfLoader } from './index';
+
+// 确保你已经在 fixtures 目录下放置了 test.pdf 文件
+const fixturePath = (filename: string) => path.join(__dirname, `./fixtures/${filename}`);
+
+let loader: FileLoaderInterface;
+
+const testFile = fixturePath('test.pdf');
+const nonExistentFile = fixturePath('nonexistent.pdf');
+
+beforeEach(() => {
+  loader = new PdfLoader();
+});
+
+describe('PdfLoader', () => {
+  it('should load pages correctly from a PDF file', async () => {
+    const pages = await loader.loadPages(testFile);
+
+    expect(pages.length).toBeGreaterThan(0);
+
+    expect(pages).toMatchSnapshot();
+  });
+
+  it('should aggregate content correctly', async () => {
+    const pages = await loader.loadPages(testFile);
+    const content = await loader.aggregateContent(pages);
+    // 默认聚合是以换行符连接各页内容
+    expect(content).toMatchSnapshot();
+  });
+
+  it('should handle file read errors in loadPages', async () => {
+    const pages = await loader.loadPages(nonExistentFile);
+    expect(pages).toHaveLength(1); // 即使失败也返回一个包含错误信息的页面
+    expect(pages[0].pageContent).toBe('');
+    expect(pages[0].metadata.error).toContain('Failed to load or parse PDF file:');
+  });
+
+  it('should attach document metadata correctly', async () => {
+    // 首先加载页面以初始化 pdfInstance，尽管此方法不直接使用页面
+    const metadata = await loader.attachDocumentMetadata!(testFile);
+
+    expect(metadata).toMatchSnapshot();
+  });
+});

package/packages/file-loaders/src/loaders/pdf/index.ts

@@ -0,0 +1,133 @@
+import { readFile } from 'node:fs/promises';
+import * as pdfjsLib from 'pdfjs-dist';
+import type { PDFDocumentProxy, PDFPageProxy, TextContent } from 'pdfjs-dist/types/src/display/api';
+
+import type { DocumentPage, FileLoaderInterface } from '../../types';
+
+/**
+ * Loads PDF files page by page using the official pdfjs-dist library.
+ */
+export class PdfLoader implements FileLoaderInterface {
+  private pdfInstance: PDFDocumentProxy | null = null;
+
+  private async getPDFFile(filePath: string) {
+    if (!!this.pdfInstance) return this.pdfInstance;
+
+    const dataBuffer = await readFile(filePath);
+
+    const loadingTask = pdfjsLib.getDocument({
+      data: new Uint8Array(dataBuffer.buffer, dataBuffer.byteOffset, dataBuffer.length),
+      useSystemFonts: true,
+      // Explicitly disable worker thread
+      worker: undefined, // Attempt to use system fonts
+    });
+
+    const pdf: PDFDocumentProxy = await loadingTask.promise;
+
+    this.pdfInstance = pdf;
+
+    return pdf;
+  }
+
+  async loadPages(filePath: string): Promise<DocumentPage[]> {
+    try {
+      const pdf: PDFDocumentProxy = await this.getPDFFile(filePath);
+
+      const pages: DocumentPage[] = [];
+
+      for (let i = 1; i <= pdf.numPages; i += 1) {
+        const page: PDFPageProxy = await pdf.getPage(i);
+        const content: TextContent = await page.getTextContent();
+
+        // --- Revert to EXACT Simple Langchain PDFLoader Logic ---
+        let lastY;
+        const textItems = [];
+        for (const item of content.items) {
+          // Ensure 'str' exists and potentially filter empty strings if needed, though Langchain's snippet doesn't explicitly filter empties
+          if ('str' in item) {
+            if (lastY === item.transform[5] || !lastY) {
+              // Exact check from Langchain
+              textItems.push(item.str);
+            } else {
+              // Exact else from Langchain for Y change
+              textItems.push(`\n${item.str}`);
+            }
+            // Update lastY, Langchain's snippet doesn't use destructuring here
+            lastY = item.transform[5];
+          }
+        }
+
+        const pageText = textItems.join(''); // Join with empty separator
+        // --- End Revert to Simple Langchain Logic ---
+
+        // Clean the final text (keep null character removal)
+        const cleanedPageContent = pageText.replaceAll('\0', '');
+
+        // Calculate stats based on the final content
+        const pageLines = cleanedPageContent.split('\n');
+        const lineCount = pageLines.length;
+        const charCount = cleanedPageContent.length;
+
+        pages.push({
+          charCount,
+          lineCount,
+          metadata: { pageNumber: i },
+          pageContent: cleanedPageContent,
+        });
+
+        // Clean up page resources
+        page.cleanup();
+      }
+
+      // Clean up document resources
+      await pdf.destroy();
+
+      return pages;
+    } catch (e) {
+      const error = e as Error;
+      console.error(
+        `Error loading PDF file ${filePath} using pdfjs-dist: ${error.message}`,
+        error.stack,
+      );
+      const errorPage: DocumentPage = {
+        charCount: 0,
+        lineCount: 0,
+        metadata: {
+          error: `Failed to load or parse PDF file: ${error.message}`,
+          filePath: filePath,
+        },
+        pageContent: '',
+      };
+      return [errorPage];
+    }
+  }
+
+  /**
+   * Aggregates content from PDF pages.
+   * Uses double newline as a separator.
+   * @param pages Array of DocumentPage objects.
+   * @returns Aggregated content as a string.
+   */
+  async aggregateContent(pages: DocumentPage[]): Promise<string> {
+    return pages
+      .filter((page) => !page.metadata.error)
+      .map((page) => page.pageContent)
+      .join('\n\n');
+  }
+
+  async attachDocumentMetadata(filePath: string): Promise<any> {
+    const pdf: PDFDocumentProxy = await this.getPDFFile(filePath);
+
+    const pdfMetadata = (await pdf.getMetadata().catch(() => null)) ?? null;
+    const pdfInfo = pdfMetadata?.info ?? {};
+    const metadata = pdfMetadata?.metadata ?? null;
+
+    return {
+      pdfInfo: pdfInfo,
+      // PDF info (Author, Title, etc.)
+      pdfMetadata: metadata,
+      // PDF metadata
+      pdfVersion: pdfjsLib.version,
+    };
+  }
+}

package/packages/file-loaders/src/loaders/pptx/__snapshots__/index.test.ts.snap

@@ -0,0 +1,40 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`PptxLoader > should aggregate content correctly (joining slides) > aggregated_content 1`] = `
+"<slide_page pageNumber="1">
+Hello
+Page1
+</slide_page>
+
+<slide_page pageNumber="2">
+Word
+Page2
+</slide_page>"
+`;
+
+exports[`PptxLoader > should load pages correctly from a PPTX file (one page per slide) 1`] = `
+[
+  {
+    "charCount": 11,
+    "lineCount": 2,
+    "metadata": {
+      "pageCount": 2,
+      "slideNumber": 1,
+      "sourceFileName": "test.pptx",
+    },
+    "pageContent": "Hello
+Page1",
+  },
+  {
+    "charCount": 10,
+    "lineCount": 2,
+    "metadata": {
+      "pageCount": 2,
+      "slideNumber": 2,
+      "sourceFileName": "test.pptx",
+    },
+    "pageContent": "Word
+Page2",
+  },
+]
+`;