@lobehub/chat 1.81.4 → 1.81.5

package/packages/file-loaders/src/loaders/pdf/index.ts

@@ -0,0 +1,133 @@
+import { readFile } from 'node:fs/promises';
+import * as pdfjsLib from 'pdfjs-dist';
+import type { PDFDocumentProxy, PDFPageProxy, TextContent } from 'pdfjs-dist/types/src/display/api';
+
+import type { DocumentPage, FileLoaderInterface } from '../../types';
+
+/**
+ * Loads PDF files page by page using the official pdfjs-dist library.
+ */
+export class PdfLoader implements FileLoaderInterface {
+  private pdfInstance: PDFDocumentProxy | null = null;
+
+  private async getPDFFile(filePath: string) {
+    if (!!this.pdfInstance) return this.pdfInstance;
+
+    const dataBuffer = await readFile(filePath);
+
+    const loadingTask = pdfjsLib.getDocument({
+      data: new Uint8Array(dataBuffer.buffer, dataBuffer.byteOffset, dataBuffer.length),
+      useSystemFonts: true,
+      // Explicitly disable worker thread
+      worker: undefined, // Attempt to use system fonts
+    });
+
+    const pdf: PDFDocumentProxy = await loadingTask.promise;
+
+    this.pdfInstance = pdf;
+
+    return pdf;
+  }
+
+  async loadPages(filePath: string): Promise<DocumentPage[]> {
+    try {
+      const pdf: PDFDocumentProxy = await this.getPDFFile(filePath);
+
+      const pages: DocumentPage[] = [];
+
+      for (let i = 1; i <= pdf.numPages; i += 1) {
+        const page: PDFPageProxy = await pdf.getPage(i);
+        const content: TextContent = await page.getTextContent();
+
+        // --- Revert to EXACT Simple Langchain PDFLoader Logic ---
+        let lastY;
+        const textItems = [];
+        for (const item of content.items) {
+          // Ensure 'str' exists and potentially filter empty strings if needed, though Langchain's snippet doesn't explicitly filter empties
+          if ('str' in item) {
+            if (lastY === item.transform[5] || !lastY) {
+              // Exact check from Langchain
+              textItems.push(item.str);
+            } else {
+              // Exact else from Langchain for Y change
+              textItems.push(`\n${item.str}`);
+            }
+            // Update lastY, Langchain's snippet doesn't use destructuring here
+            lastY = item.transform[5];
+          }
+        }
+
+        const pageText = textItems.join(''); // Join with empty separator
+        // --- End Revert to Simple Langchain Logic ---
+
+        // Clean the final text (keep null character removal)
+        const cleanedPageContent = pageText.replaceAll('\0', '');
+
+        // Calculate stats based on the final content
+        const pageLines = cleanedPageContent.split('\n');
+        const lineCount = pageLines.length;
+        const charCount = cleanedPageContent.length;
+
+        pages.push({
+          charCount,
+          lineCount,
+          metadata: { pageNumber: i },
+          pageContent: cleanedPageContent,
+        });
+
+        // Clean up page resources
+        page.cleanup();
+      }
+
+      // Clean up document resources
+      await pdf.destroy();
+
+      return pages;
+    } catch (e) {
+      const error = e as Error;
+      console.error(
+        `Error loading PDF file ${filePath} using pdfjs-dist: ${error.message}`,
+        error.stack,
+      );
+      const errorPage: DocumentPage = {
+        charCount: 0,
+        lineCount: 0,
+        metadata: {
+          error: `Failed to load or parse PDF file: ${error.message}`,
+          filePath: filePath,
+        },
+        pageContent: '',
+      };
+      return [errorPage];
+    }
+  }
+
+  /**
+   * Aggregates content from PDF pages.
+   * Uses double newline as a separator.
+   * @param pages Array of DocumentPage objects.
+   * @returns Aggregated content as a string.
+   */
+  async aggregateContent(pages: DocumentPage[]): Promise<string> {
+    return pages
+      .filter((page) => !page.metadata.error)
+      .map((page) => page.pageContent)
+      .join('\n\n');
+  }
+
+  async attachDocumentMetadata(filePath: string): Promise<any> {
+    const pdf: PDFDocumentProxy = await this.getPDFFile(filePath);
+
+    const pdfMetadata = (await pdf.getMetadata().catch(() => null)) ?? null;
+    const pdfInfo = pdfMetadata?.info ?? {};
+    const metadata = pdfMetadata?.metadata ?? null;
+
+    return {
+      pdfInfo: pdfInfo,
+      // PDF info (Author, Title, etc.)
+      pdfMetadata: metadata,
+      // PDF metadata
+      pdfVersion: pdfjsLib.version,
+    };
+  }
+}

package/packages/file-loaders/src/loaders/pptx/__snapshots__/index.test.ts.snap

@@ -0,0 +1,40 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`PptxLoader > should aggregate content correctly (joining slides) > aggregated_content 1`] = `
+"<slide_page pageNumber="1">
+Hello
+Page1
+</slide_page>
+
+<slide_page pageNumber="2">
+Word
+Page2
+</slide_page>"
+`;
+
+exports[`PptxLoader > should load pages correctly from a PPTX file (one page per slide) 1`] = `
+[
+  {
+    "charCount": 11,
+    "lineCount": 2,
+    "metadata": {
+      "pageCount": 2,
+      "slideNumber": 1,
+      "sourceFileName": "test.pptx",
+    },
+    "pageContent": "Hello
+Page1",
+  },
+  {
+    "charCount": 10,
+    "lineCount": 2,
+    "metadata": {
+      "pageCount": 2,
+      "slideNumber": 2,
+      "sourceFileName": "test.pptx",
+    },
+    "pageContent": "Word
+Page2",
+  },
+]
+`;

package/packages/file-loaders/src/loaders/pptx/index.test.ts

@@ -0,0 +1,47 @@
+// @vitest-environment node
+import path from 'node:path';
+import { beforeEach, describe, expect, it } from 'vitest';
+
+import type { FileLoaderInterface } from '../../types';
+import { PptxLoader } from './index';
+
+// Import PptxLoader
+
+// 确保你已经在 fixtures 目录下放置了 test.pptx 文件
+// 这个 PPTX 文件最好包含多个幻灯片 (slides) 以便测试
+const fixturePath = (filename: string) => path.join(__dirname, `./fixtures/${filename}`);
+
+let loader: FileLoaderInterface;
+
+const testFile = fixturePath('test.pptx'); // Use .pptx
+const nonExistentFile = fixturePath('nonexistent.pptx'); // Use .pptx
+
+beforeEach(() => {
+  loader = new PptxLoader(); // Instantiate PptxLoader
+});
+
+describe('PptxLoader', () => {
+  // Describe PptxLoader
+  it('should load pages correctly from a PPTX file (one page per slide)', async () => {
+    const pages = await loader.loadPages(testFile);
+    // PPTX 文件有多少个 slide，就应该有多少个 page
+    expect(pages.length).toBeGreaterThan(1);
+
+    // 直接对整个 pages 数组进行快照测试 (会包含 slideNumber)
+    expect(pages).toMatchSnapshot();
+  });
+
+  it('should aggregate content correctly (joining slides)', async () => {
+    const pages = await loader.loadPages(testFile);
+    const content = await loader.aggregateContent(pages);
+    // 默认聚合是以换行符连接各 slide 内容
+    expect(content).toMatchSnapshot('aggregated_content');
+  });
+
+  it('should handle file read errors in loadPages', async () => {
+    const pages = await loader.loadPages(nonExistentFile);
+    expect(pages).toHaveLength(1); // 即使失败也返回一个包含错误信息的页面
+    expect(pages[0].pageContent).toBe('');
+    expect(pages[0].metadata.error).toContain('Failed to load or process PPTX file:'); // Update error message check
+  });
+});

package/packages/file-loaders/src/loaders/pptx/index.ts

@@ -0,0 +1,186 @@
+import path from 'node:path';
+
+import type { DocumentPage, FileLoaderInterface } from '../../types';
+import { type ExtractedFile, extractFiles, parseString } from '../../utils/parser-utils';
+
+/**
+ * Represents a loader for PPTX files using extracted utility functions.
+ *
+ * This loader reads a PPTX file, extracts text content from each slide,
+ * and represents each slide as a `DocumentPage`.
+ */
+export class PptxLoader implements FileLoaderInterface {
+  /**
+   * Loads pages from the specified PPTX file path.
+   *
+   * @param filePath The absolute path to the PPTX file.
+   * @returns A Promise resolving to an array of `DocumentPage` objects.
+   *          If loading or parsing fails, it returns an array containing a single
+   *          `DocumentPage` object with error information in its metadata.
+   */
+  async loadPages(filePath: string): Promise<DocumentPage[]> {
+    const sourceFileName = path.basename(filePath);
+
+    try {
+      // --- File Extraction Step ---
+      const slidesRegex = /ppt\/slides\/slide\d+\.xml/g;
+      const slideNumberRegex = /slide(\d+)\.xml/;
+
+      // Extract only slide XML files
+      const slideFiles: ExtractedFile[] = await extractFiles(filePath, (fileName) =>
+        slidesRegex.test(fileName),
+      );
+
+      // --- Validation Step ---
+      if (slideFiles.length === 0) {
+        console.warn(`No slide XML files found in ${sourceFileName}. May be corrupted or empty.`);
+        return [
+          this.createErrorPage(
+            'No slides found. The PPTX file might be empty, corrupted, or does not contain standard slide XMLs.',
+            sourceFileName,
+          ),
+        ];
+      }
+
+      // --- Sorting Step ---
+      // Sort files based on the slide number extracted from the path
+      slideFiles.sort((a, b) => {
+        const matchA = a.path.match(slideNumberRegex);
+        const matchB = b.path.match(slideNumberRegex);
+        const numA = matchA ? parseInt(matchA[1], 10) : Infinity;
+        const numB = matchB ? parseInt(matchB[1], 10) : Infinity;
+        return numA - numB;
+      });
+
+      // --- Page Creation Step ---
+      const pages: DocumentPage[] = slideFiles
+        .map((slideFile, index) => {
+          try {
+            const xmlDoc = parseString(slideFile.content);
+            const paragraphNodes = xmlDoc.getElementsByTagName('a:p');
+
+            const slideText = Array.from(paragraphNodes)
+              .map((pNode) => {
+                const textNodes = pNode.getElementsByTagName('a:t');
+                return Array.from(textNodes)
+                  .map((tNode) => (tNode.childNodes[0] ? tNode.childNodes[0].nodeValue : ''))
+                  .join(''); // Join text within a paragraph without spaces
+              })
+              .filter((text) => text.length > 0) // Filter out empty paragraphs
+              .join('\n'); // Join paragraphs with newline
+
+            const lines = slideText.split('\n');
+            const slideNumberMatch = slideFile.path.match(slideNumberRegex);
+            const slideNumber = slideNumberMatch ? parseInt(slideNumberMatch[1], 10) : index + 1; // Fallback to index if regex fails
+
+            const metadata = {
+              pageCount: slideFiles.length, // Total number of slides found
+              slideNumber: slideNumber,
+              sourceFileName,
+            };
+
+            return {
+              charCount: slideText.length,
+              lineCount: lines.length,
+              metadata: metadata,
+              pageContent: slideText.trim(), // Trim final content
+            };
+          } catch (parseError) {
+            console.error(
+              `Failed to parse XML for slide ${slideFile.path} in ${sourceFileName}: ${parseError instanceof Error ? parseError.message : String(parseError)}`,
+            );
+            // Create a specific error page for this slide, or could return null and filter later
+            // Returning null might be better if one slide fails but others succeed.
+            // For now, let's keep it simple and create an error page for this slide.
+            return this.createErrorPage(
+              `Error parsing slide ${slideFile.path}: ${parseError instanceof Error ? parseError.message : String(parseError)}`,
+              sourceFileName,
+              slideFile.path,
+            );
+          }
+        })
+        // Filter out any potential nulls if we change the error handling above
+        .filter((page): page is DocumentPage => page !== null);
+
+      if (pages.length === 0) {
+        // This case might happen if all slides failed to parse
+        console.warn(`Parsing resulted in zero valid pages for ${sourceFileName}`);
+        return [this.createErrorPage('Parsing resulted in zero valid pages.', sourceFileName)];
+      }
+
+      // Check if all pages are error pages
+      const allErrored = pages.every((page) => page.metadata?.error);
+      if (allErrored) {
+        // If all pages resulted in errors, perhaps return a single summary error
+        console.warn(`All slides failed to parse for ${sourceFileName}`);
+        return [this.createErrorPage('All slides failed to parse correctly.', sourceFileName)];
+        // Or return all the individual error pages: return pages;
+      }
+
+      return pages;
+    } catch (error) {
+      // --- Error Handling Step ---
+      // This catches errors from extractFiles or other unexpected issues
+      const errorMessage = `Failed to load or process PPTX file: ${error instanceof Error ? error.message : String(error)}`;
+      console.error(errorMessage, { filePath });
+      return [this.createErrorPage(errorMessage, sourceFileName)];
+    }
+  }
+
+  /**
+   * Aggregates the content from all DocumentPages (slides).
+   *
+   * Prepends each slide's content with a "## Slide: N" header.
+   * Joins the content of slides with a standard separator.
+   *
+   * @param pages An array of `DocumentPage` objects obtained from `loadPages`.
+   * @returns A Promise resolving to the aggregated content string.
+   */
+  async aggregateContent(pages: DocumentPage[]): Promise<string> {
+    // Ensure pages array is valid and non-empty before proceeding
+    // Filter out error pages before aggregation unless we want to include error messages
+    const validPages = pages.filter((page) => !page.metadata?.error);
+
+    if (validPages.length === 0) {
+      // If only error pages existed, return empty or a summary error message
+      return pages[0]?.pageContent || ''; // Return content of the first page (might be an error page)
+    }
+
+    return validPages
+      .map((page) => {
+        const slideNumber = page.metadata?.slideNumber;
+        // Use Markdown H2 for slide headers
+        const header = slideNumber ? `<slide_page pageNumber="${slideNumber}">` : '<slide_page>'; // Fallback header
+        return `${header}
+${page.pageContent}
+</slide_page>`;
+      })
+      .join('\n\n'); // Use Markdown horizontal rule as separator
+  }
+
+  /**
+   * Helper method to create a standardized error page object.
+   *
+   * @param errorInfo A string describing the error.
+   * @param sourceFileName The name of the file that caused the error.
+   * @param sourceFilePath Optional: Specific path within the archive that caused the error (e.g., slide path)
+   * @returns A `DocumentPage` object representing the error state.
+   */
+  private createErrorPage(
+    errorInfo: string,
+    sourceFileName: string,
+    sourceFilePath?: string,
+  ): DocumentPage {
+    return {
+      charCount: 0,
+      lineCount: 0,
+      metadata: {
+        error: errorInfo,
+        pageCount: 0,
+        sourceFileName: sourceFileName,
+        ...(sourceFilePath && { sourceFilePath }), // Add specific path if available
+      },
+      pageContent: '', // Error pages have no content
+    };
+  }
+}

package/packages/file-loaders/src/loaders/text/__snapshots__/index.test.ts.snap

@@ -0,0 +1,15 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`TextLoader > should load pages correctly 1`] = `
+{
+  "charCount": 25,
+  "lineCount": 3,
+  "metadata": {
+    "lineNumberEnd": 3,
+    "lineNumberStart": 1,
+  },
+  "pageContent": "Hello Text.
+Second Line.
+",
+}
+`;

package/packages/file-loaders/src/loaders/text/fixtures/test.txt

@@ -0,0 +1,2 @@
+Hello Text.
+Second Line.

package/packages/file-loaders/src/loaders/text/index.test.ts

@@ -0,0 +1,38 @@
+import path from 'node:path';
+import { beforeEach } from 'vitest';
+
+import type { FileLoaderInterface } from '../../types';
+import { TextLoader } from './index';
+
+const fixturePath = (filename: string) => path.join(__dirname, `./fixtures/${filename}`);
+
+let loader: FileLoaderInterface;
+
+const testFile = fixturePath('test.txt');
+
+beforeEach(() => {
+  loader = new TextLoader();
+});
+
+describe('TextLoader', () => {
+  it('should load pages correctly', async () => {
+    const pages = await loader.loadPages(testFile);
+    expect(pages).toHaveLength(1);
+    const page = pages[0];
+    expect(page).toMatchSnapshot();
+  });
+
+  it('should aggregate content correctly', async () => {
+    const pages = await loader.loadPages(testFile);
+    const content = await loader.aggregateContent(pages);
+    // Default aggregation joins with newline
+    expect(content).toBe('Hello Text.\nSecond Line.\n');
+  });
+
+  it('should handle file read errors in loadPages', async () => {
+    const pages = await loader.loadPages(fixturePath('nonexistent.txt'));
+    expect(pages).toHaveLength(1);
+    expect(pages[0].metadata.error).toContain('Failed to load text file');
+    expect(pages[0].pageContent).toBe('');
+  });
+});

package/packages/file-loaders/src/loaders/text/index.ts

@@ -0,0 +1,53 @@
+import { readFile } from 'node:fs/promises';
+
+import type { DocumentPage, FileLoaderInterface } from '../../types';
+
+/**
+ * 用于加载纯文本文件的加载器。
+ */
+export class TextLoader implements FileLoaderInterface {
+  async loadPages(filePath: string): Promise<DocumentPage[]> {
+    try {
+      const fileContent = await readFile(filePath, 'utf8');
+      const lines = fileContent.split('\n');
+      const lineCount = lines.length;
+      const charCount = fileContent.length;
+
+      const page: DocumentPage = {
+        charCount,
+        lineCount,
+        metadata: {
+          lineNumberEnd: lineCount,
+          lineNumberStart: 1,
+        },
+        pageContent: fileContent,
+      };
+
+      return [page];
+    } catch (e) {
+      const error = e as Error;
+      console.error(`Error loading text file ${filePath}: ${error.message}`);
+      // 如果读取失败，返回一个包含错误信息的 Page
+      const errorPage: DocumentPage = {
+        charCount: 0,
+        lineCount: 0,
+        metadata: {
+          error: `Failed to load text file: ${error.message}`,
+        },
+        pageContent: '',
+      };
+      return [errorPage];
+    }
+  }
+
+  /**
+   * 对于纯文本，简单地连接所有页面的内容。
+   * （虽然 TextLoader 通常只有一个页面，但保持接口一致性）
+   * @param pages 页面数组
+   * @returns 聚合后的内容
+   */
+  async aggregateContent(pages: DocumentPage[]): Promise<string> {
+    // 默认使用换行符连接，可以根据需要调整或使其可配置
+    return pages.map((page) => page.pageContent).join('\n');
+  }
+}