@lobehub/chat 1.81.4 → 1.81.6

package/packages/file-loaders/src/loaders/pptx/index.test.ts

@@ -0,0 +1,47 @@
+// @vitest-environment node
+import path from 'node:path';
+import { beforeEach, describe, expect, it } from 'vitest';
+
+import type { FileLoaderInterface } from '../../types';
+import { PptxLoader } from './index';
+
+// Import PptxLoader
+
+// 确保你已经在 fixtures 目录下放置了 test.pptx 文件
+// 这个 PPTX 文件最好包含多个幻灯片 (slides) 以便测试
+const fixturePath = (filename: string) => path.join(__dirname, `./fixtures/${filename}`);
+
+let loader: FileLoaderInterface;
+
+const testFile = fixturePath('test.pptx'); // Use .pptx
+const nonExistentFile = fixturePath('nonexistent.pptx'); // Use .pptx
+
+beforeEach(() => {
+  loader = new PptxLoader(); // Instantiate PptxLoader
+});
+
+describe('PptxLoader', () => {
+  // Describe PptxLoader
+  it('should load pages correctly from a PPTX file (one page per slide)', async () => {
+    const pages = await loader.loadPages(testFile);
+    // PPTX 文件有多少个 slide，就应该有多少个 page
+    expect(pages.length).toBeGreaterThan(1);
+
+    // 直接对整个 pages 数组进行快照测试 (会包含 slideNumber)
+    expect(pages).toMatchSnapshot();
+  });
+
+  it('should aggregate content correctly (joining slides)', async () => {
+    const pages = await loader.loadPages(testFile);
+    const content = await loader.aggregateContent(pages);
+    // 默认聚合是以换行符连接各 slide 内容
+    expect(content).toMatchSnapshot('aggregated_content');
+  });
+
+  it('should handle file read errors in loadPages', async () => {
+    const pages = await loader.loadPages(nonExistentFile);
+    expect(pages).toHaveLength(1); // 即使失败也返回一个包含错误信息的页面
+    expect(pages[0].pageContent).toBe('');
+    expect(pages[0].metadata.error).toContain('Failed to load or process PPTX file:'); // Update error message check
+  });
+});

package/packages/file-loaders/src/loaders/pptx/index.ts

@@ -0,0 +1,186 @@
+import path from 'node:path';
+
+import type { DocumentPage, FileLoaderInterface } from '../../types';
+import { type ExtractedFile, extractFiles, parseString } from '../../utils/parser-utils';
+
+/**
+ * Represents a loader for PPTX files using extracted utility functions.
+ *
+ * This loader reads a PPTX file, extracts text content from each slide,
+ * and represents each slide as a `DocumentPage`.
+ */
+export class PptxLoader implements FileLoaderInterface {
+  /**
+   * Loads pages from the specified PPTX file path.
+   *
+   * @param filePath The absolute path to the PPTX file.
+   * @returns A Promise resolving to an array of `DocumentPage` objects.
+   *          If loading or parsing fails, it returns an array containing a single
+   *          `DocumentPage` object with error information in its metadata.
+   */
+  async loadPages(filePath: string): Promise<DocumentPage[]> {
+    const sourceFileName = path.basename(filePath);
+
+    try {
+      // --- File Extraction Step ---
+      const slidesRegex = /ppt\/slides\/slide\d+\.xml/g;
+      const slideNumberRegex = /slide(\d+)\.xml/;
+
+      // Extract only slide XML files
+      const slideFiles: ExtractedFile[] = await extractFiles(filePath, (fileName) =>
+        slidesRegex.test(fileName),
+      );
+
+      // --- Validation Step ---
+      if (slideFiles.length === 0) {
+        console.warn(`No slide XML files found in ${sourceFileName}. May be corrupted or empty.`);
+        return [
+          this.createErrorPage(
+            'No slides found. The PPTX file might be empty, corrupted, or does not contain standard slide XMLs.',
+            sourceFileName,
+          ),
+        ];
+      }
+
+      // --- Sorting Step ---
+      // Sort files based on the slide number extracted from the path
+      slideFiles.sort((a, b) => {
+        const matchA = a.path.match(slideNumberRegex);
+        const matchB = b.path.match(slideNumberRegex);
+        const numA = matchA ? parseInt(matchA[1], 10) : Infinity;
+        const numB = matchB ? parseInt(matchB[1], 10) : Infinity;
+        return numA - numB;
+      });
+
+      // --- Page Creation Step ---
+      const pages: DocumentPage[] = slideFiles
+        .map((slideFile, index) => {
+          try {
+            const xmlDoc = parseString(slideFile.content);
+            const paragraphNodes = xmlDoc.getElementsByTagName('a:p');
+
+            const slideText = Array.from(paragraphNodes)
+              .map((pNode) => {
+                const textNodes = pNode.getElementsByTagName('a:t');
+                return Array.from(textNodes)
+                  .map((tNode) => (tNode.childNodes[0] ? tNode.childNodes[0].nodeValue : ''))
+                  .join(''); // Join text within a paragraph without spaces
+              })
+              .filter((text) => text.length > 0) // Filter out empty paragraphs
+              .join('\n'); // Join paragraphs with newline
+
+            const lines = slideText.split('\n');
+            const slideNumberMatch = slideFile.path.match(slideNumberRegex);
+            const slideNumber = slideNumberMatch ? parseInt(slideNumberMatch[1], 10) : index + 1; // Fallback to index if regex fails
+
+            const metadata = {
+              pageCount: slideFiles.length, // Total number of slides found
+              slideNumber: slideNumber,
+              sourceFileName,
+            };
+
+            return {
+              charCount: slideText.length,
+              lineCount: lines.length,
+              metadata: metadata,
+              pageContent: slideText.trim(), // Trim final content
+            };
+          } catch (parseError) {
+            console.error(
+              `Failed to parse XML for slide ${slideFile.path} in ${sourceFileName}: ${parseError instanceof Error ? parseError.message : String(parseError)}`,
+            );
+            // Create a specific error page for this slide, or could return null and filter later
+            // Returning null might be better if one slide fails but others succeed.
+            // For now, let's keep it simple and create an error page for this slide.
+            return this.createErrorPage(
+              `Error parsing slide ${slideFile.path}: ${parseError instanceof Error ? parseError.message : String(parseError)}`,
+              sourceFileName,
+              slideFile.path,
+            );
+          }
+        })
+        // Filter out any potential nulls if we change the error handling above
+        .filter((page): page is DocumentPage => page !== null);
+
+      if (pages.length === 0) {
+        // This case might happen if all slides failed to parse
+        console.warn(`Parsing resulted in zero valid pages for ${sourceFileName}`);
+        return [this.createErrorPage('Parsing resulted in zero valid pages.', sourceFileName)];
+      }
+
+      // Check if all pages are error pages
+      const allErrored = pages.every((page) => page.metadata?.error);
+      if (allErrored) {
+        // If all pages resulted in errors, perhaps return a single summary error
+        console.warn(`All slides failed to parse for ${sourceFileName}`);
+        return [this.createErrorPage('All slides failed to parse correctly.', sourceFileName)];
+        // Or return all the individual error pages: return pages;
+      }
+
+      return pages;
+    } catch (error) {
+      // --- Error Handling Step ---
+      // This catches errors from extractFiles or other unexpected issues
+      const errorMessage = `Failed to load or process PPTX file: ${error instanceof Error ? error.message : String(error)}`;
+      console.error(errorMessage, { filePath });
+      return [this.createErrorPage(errorMessage, sourceFileName)];
+    }
+  }
+
+  /**
+   * Aggregates the content from all DocumentPages (slides).
+   *
+   * Prepends each slide's content with a "## Slide: N" header.
+   * Joins the content of slides with a standard separator.
+   *
+   * @param pages An array of `DocumentPage` objects obtained from `loadPages`.
+   * @returns A Promise resolving to the aggregated content string.
+   */
+  async aggregateContent(pages: DocumentPage[]): Promise<string> {
+    // Ensure pages array is valid and non-empty before proceeding
+    // Filter out error pages before aggregation unless we want to include error messages
+    const validPages = pages.filter((page) => !page.metadata?.error);
+
+    if (validPages.length === 0) {
+      // If only error pages existed, return empty or a summary error message
+      return pages[0]?.pageContent || ''; // Return content of the first page (might be an error page)
+    }
+
+    return validPages
+      .map((page) => {
+        const slideNumber = page.metadata?.slideNumber;
+        // Use Markdown H2 for slide headers
+        const header = slideNumber ? `<slide_page pageNumber="${slideNumber}">` : '<slide_page>'; // Fallback header
+        return `${header}
+${page.pageContent}
+</slide_page>`;
+      })
+      .join('\n\n'); // Use Markdown horizontal rule as separator
+  }
+
+  /**
+   * Helper method to create a standardized error page object.
+   *
+   * @param errorInfo A string describing the error.
+   * @param sourceFileName The name of the file that caused the error.
+   * @param sourceFilePath Optional: Specific path within the archive that caused the error (e.g., slide path)
+   * @returns A `DocumentPage` object representing the error state.
+   */
+  private createErrorPage(
+    errorInfo: string,
+    sourceFileName: string,
+    sourceFilePath?: string,
+  ): DocumentPage {
+    return {
+      charCount: 0,
+      lineCount: 0,
+      metadata: {
+        error: errorInfo,
+        pageCount: 0,
+        sourceFileName: sourceFileName,
+        ...(sourceFilePath && { sourceFilePath }), // Add specific path if available
+      },
+      pageContent: '', // Error pages have no content
+    };
+  }
+}

package/packages/file-loaders/src/loaders/text/__snapshots__/index.test.ts.snap

@@ -0,0 +1,15 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`TextLoader > should load pages correctly 1`] = `
+{
+  "charCount": 25,
+  "lineCount": 3,
+  "metadata": {
+    "lineNumberEnd": 3,
+    "lineNumberStart": 1,
+  },
+  "pageContent": "Hello Text.
+Second Line.
+",
+}
+`;

package/packages/file-loaders/src/loaders/text/fixtures/test.txt

@@ -0,0 +1,2 @@
+Hello Text.
+Second Line.

package/packages/file-loaders/src/loaders/text/index.test.ts

@@ -0,0 +1,38 @@
+import path from 'node:path';
+import { beforeEach } from 'vitest';
+
+import type { FileLoaderInterface } from '../../types';
+import { TextLoader } from './index';
+
+const fixturePath = (filename: string) => path.join(__dirname, `./fixtures/${filename}`);
+
+let loader: FileLoaderInterface;
+
+const testFile = fixturePath('test.txt');
+
+beforeEach(() => {
+  loader = new TextLoader();
+});
+
+describe('TextLoader', () => {
+  it('should load pages correctly', async () => {
+    const pages = await loader.loadPages(testFile);
+    expect(pages).toHaveLength(1);
+    const page = pages[0];
+    expect(page).toMatchSnapshot();
+  });
+
+  it('should aggregate content correctly', async () => {
+    const pages = await loader.loadPages(testFile);
+    const content = await loader.aggregateContent(pages);
+    // Default aggregation joins with newline
+    expect(content).toBe('Hello Text.\nSecond Line.\n');
+  });
+
+  it('should handle file read errors in loadPages', async () => {
+    const pages = await loader.loadPages(fixturePath('nonexistent.txt'));
+    expect(pages).toHaveLength(1);
+    expect(pages[0].metadata.error).toContain('Failed to load text file');
+    expect(pages[0].pageContent).toBe('');
+  });
+});

package/packages/file-loaders/src/loaders/text/index.ts

@@ -0,0 +1,53 @@
+import { readFile } from 'node:fs/promises';
+
+import type { DocumentPage, FileLoaderInterface } from '../../types';
+
+/**
+ * 用于加载纯文本文件的加载器。
+ */
+export class TextLoader implements FileLoaderInterface {
+  async loadPages(filePath: string): Promise<DocumentPage[]> {
+    try {
+      const fileContent = await readFile(filePath, 'utf8');
+      const lines = fileContent.split('\n');
+      const lineCount = lines.length;
+      const charCount = fileContent.length;
+
+      const page: DocumentPage = {
+        charCount,
+        lineCount,
+        metadata: {
+          lineNumberEnd: lineCount,
+          lineNumberStart: 1,
+        },
+        pageContent: fileContent,
+      };
+
+      return [page];
+    } catch (e) {
+      const error = e as Error;
+      console.error(`Error loading text file ${filePath}: ${error.message}`);
+      // 如果读取失败，返回一个包含错误信息的 Page
+      const errorPage: DocumentPage = {
+        charCount: 0,
+        lineCount: 0,
+        metadata: {
+          error: `Failed to load text file: ${error.message}`,
+        },
+        pageContent: '',
+      };
+      return [errorPage];
+    }
+  }
+
+  /**
+   * 对于纯文本，简单地连接所有页面的内容。
+   * （虽然 TextLoader 通常只有一个页面，但保持接口一致性）
+   * @param pages 页面数组
+   * @returns 聚合后的内容
+   */
+  async aggregateContent(pages: DocumentPage[]): Promise<string> {
+    // 默认使用换行符连接，可以根据需要调整或使其可配置
+    return pages.map((page) => page.pageContent).join('\n');
+  }
+}

package/packages/file-loaders/src/types.ts

@@ -0,0 +1,200 @@
+// Define supported file types - consider using an enum or const assertion
+export type SupportedFileType = 'pdf' | 'docx' | 'txt' | 'excel' | 'pptx'; // | 'pptx' | 'latex' | 'epub' | 'code' | 'markdown';
+
+/**
+ * 代表一个完整的已加载文件，包含文件级信息和其所有页面/块。
+ */
+export interface FileDocument {
+  /**
+   * 文件内容
+   */
+  content: string;
+
+  /**
+   * 文件创建时间戳。
+   */
+  createdTime: Date;
+
+  /**
+   * 文件类型或扩展名。
+   */
+  fileType: string;
+
+  /**
+   * 原始文件名。
+   */
+  filename: string;
+
+  /**
+   * 文件级别的元数据。
+   * 例如从文件属性中提取的标题、作者，或整个文件加载失败时的错误。
+   */
+  metadata: {
+    /**
+     * 允许添加其他文件级别的元数据。
+     */
+    [key: string]: any;
+    /**
+     * 文档作者 (如果可用)。
+     */
+    author?: string;
+    /**
+     * 如果整个文件加载失败，记录错误信息。
+     */
+    error?: string;
+    /**
+     * 文档标题 (如果可用)。
+     */
+    title?: string;
+  };
+
+  /**
+   * 文件最后修改时间戳。
+   */
+  modifiedTime: Date;
+
+  /**
+   * 包含文档中所有逻辑页面/块的数组。
+   * 顺序通常对应文件中的自然顺序。
+   */
+  pages?: DocumentPage[];
+
+  /**
+   * 原始文件的完整路径。
+   */
+  source: string;
+
+  /**
+   * 整个文档的总字符数 (所有 Page 的 charCount 之和)。
+   * 需要在所有 Page 加载和计算后得出。
+   */
+  totalCharCount: number;
+
+  /**
+   * 整个文档的总行数 (所有 Page 的 lineCount 之和)。
+   * 需要在所有 Page 加载和计算后得出。
+   */
+  totalLineCount: number;
+}
+
+/**
+ * 代表文件中的一个逻辑单元/页面/块。
+ */
+export interface DocumentPage {
+  /**
+   * 此页/块内容的字符数。
+   */
+  charCount: number;
+
+  /**
+   * 此页/块内容的行数。
+   */
+  lineCount: number;
+
+  /**
+   * 与此页/块相关的元数据。
+   */
+  metadata: {
+    /**
+     * 允许添加其他特定于页/块的元数据。
+     */
+    [key: string]: any;
+
+    /**
+     * 如果原始文件单元被进一步分割成块，这是当前块的索引。
+     */
+    chunkIndex?: number;
+
+    /**
+     * 处理此页/块时发生的错误。
+     */
+    error?: string;
+
+    /**
+     * 此页/块在原始文件中的结束行号。
+     */
+    lineNumberEnd?: number;
+
+    /**
+     * 此页/块在原始文件中的起始行号。
+     */
+    lineNumberStart?: number;
+
+    /**
+     * 页码 (适用于 PDF, DOCX)。
+     */
+    pageNumber?: number;
+
+    /**
+     * 与此页/块相关的章节标题。
+     */
+    sectionTitle?: string;
+
+    /**
+     * 工作表名称 (适用于 XLSX)。
+     */
+    sheetName?: string;
+
+    /**
+     * 幻灯片编号 (适用于 PPTX)。
+     */
+    slideNumber?: number;
+
+    /**
+     * 如果原始文件单元被进一步分割成块，这是该单元的总块数。
+     */
+    totalChunks?: number;
+  };
+
+  /**
+   * 此页/块的核心文本内容。
+   */
+  pageContent: string;
+}
+
+/**
+ * 可选的文件元数据，用于覆盖从文件系统读取的信息。
+ */
+export interface FileMetadata {
+  /**
+   * 文件创建时间戳。
+   */
+  createdTime?: Date;
+  /**
+   * 文件类型或扩展名。
+   */
+  fileType?: string;
+  /**
+   * 文件名。
+   */
+  filename?: string;
+  /**
+   * 文件最后修改时间戳。
+   */
+  modifiedTime?: Date;
+  /**
+   * 文件来源标识 (例如 S3 URL 或原始路径)。
+   */
+  source?: string;
+}
+
+/**
+ * 定义所有文件加载器类必须实现的接口。
+ */
+export interface FileLoaderInterface {
+  /**
+   * 将从 loadPages 获取的页面内容聚合成单一的字符串。
+   * @param pages DocumentPage 对象的数组。
+   * @returns 返回聚合后的文本内容的 Promise。
+   */
+  aggregateContent(pages: DocumentPage[]): Promise<string>;
+
+  attachDocumentMetadata?(filePath: string): Promise<Record<string, any>>;
+
+  /**
+   * 根据文件路径加载文件内容，并将其分割为逻辑页面/块。
+   * @param filePath 文件的完整路径。
+   * @returns 返回包含 DocumentPage 对象的数组的 Promise。
+   */
+  loadPages(filePath: string): Promise<DocumentPage[]>;
+}

package/packages/file-loaders/src/utils/isTextReadableFile.ts

@@ -0,0 +1,68 @@
+export const TEXT_READABLE_FILE_TYPES = [
+  // Plain Text & Markup
+  'txt',
+  'md',
+  'markdown',
+  'mdx',
+
+  // Configuration & Data
+  'json',
+  'xml',
+  'yaml',
+  'yml',
+  'toml',
+  'ini',
+  'cfg',
+  'conf',
+  'csv',
+
+  // Web Development
+  'html',
+  'htm',
+  'css',
+  'scss',
+  'less',
+  'js',
+  'jsx',
+  'ts',
+  'tsx',
+  'mjs',
+  'vue',
+  'svelte',
+  'svg',
+
+  // Scripting & Programming
+  'php',
+  'py',
+  'rb',
+  'java',
+  'c',
+  'cpp',
+  'h',
+  'hpp',
+  'cs',
+  'go',
+  'rs',
+  'swift',
+  'kt',
+  'sh',
+  'bash',
+  'bat',
+  'ps1',
+
+  // Other
+  'log',
+  'sql',
+  'patch',
+  'diff',
+  'db', // Often text-based, like SQLite journals
+];
+
+/**
+ * Determine if a file can be read as text based on its extension.
+ * @param fileType File extension (without the leading dot)
+ * @returns Whether the file is likely text-readable
+ */
+export function isTextReadableFile(fileType: string): boolean {
+  return TEXT_READABLE_FILE_TYPES.includes(fileType.toLowerCase());
+}