@lobehub/chat 1.136.11 → 1.136.13

package/packages/file-loaders/src/loaders/docx/index.ts

@@ -1,70 +1,61 @@
-import { DocxLoader as LangchainDocxLoader } from '@langchain/community/document_loaders/fs/docx';
 import debug from 'debug';
+import fs from 'node:fs/promises';
+import mammoth from 'mammoth';
 
 import type { DocumentPage, FileLoaderInterface } from '../../types';
 
 const log = debug('file-loaders:docx');
 
 /**
- * Loads Word documents (.docx) using the LangChain Community DocxLoader.
+ * Loads Word documents (.docx) using mammoth library.
+ * Extracts text content and basic metadata from DOCX files.
  */
 export class DocxLoader implements FileLoaderInterface {
   async loadPages(filePath: string): Promise<DocumentPage[]> {
     log('Loading DOCX file:', filePath);
     try {
-      let loader: LangchainDocxLoader;
-      if (filePath.endsWith('.doc')) {
-        loader = new LangchainDocxLoader(filePath, { type: 'doc' });
-      } else {
-        loader = new LangchainDocxLoader(filePath, { type: 'docx' });
-      }
-      log('LangChain DocxLoader created');
-      const docs = await loader.load(); // Langchain DocxLoader typically loads the whole doc as one
-      log('DOCX document loaded, parts:', docs.length);
-
-      const pages: DocumentPage[] = docs.map((doc) => {
-        const pageContent = doc.pageContent || '';
-        const lines = pageContent.split('\n');
-        const lineCount = lines.length;
-        const charCount = pageContent.length;
+      // Read file as buffer
+      const buffer = await fs.readFile(filePath);
+      log('File buffer read, size:', buffer.length);
 
-        // Langchain DocxLoader doesn't usually provide page numbers in metadata
-        // We treat it as a single page
-        const metadata = {
-          ...doc.metadata, // Include any other metadata Langchain provides
-          pageNumber: 1,
-        };
+      // Extract text using mammoth
+      const result = await mammoth.extractRawText({ buffer });
+      const pageContent = result.value;
+      log('Text extracted, length:', pageContent.length);
 
-        // @ts-expect-error Remove source if present, as it's handled at the FileDocument level
-        delete metadata.source;
+      // Count lines and characters
+      const lines = pageContent.split('\n');
+      const lineCount = lines.length;
+      const charCount = pageContent.length;
 
-        log('DOCX document processed, lines:', lineCount, 'chars:', charCount);
+      log('DOCX document processed, lines:', lineCount, 'chars:', charCount);
 
-        return {
-          charCount,
-          lineCount,
-          metadata,
-          pageContent,
-        };
-      });
+      // Create single page with extracted content
+      const page: DocumentPage = {
+        charCount,
+        lineCount,
+        metadata: {
+          pageNumber: 1,
+        },
+        pageContent,
+      };
 
-      // If docs array is empty (e.g., empty file), create an empty page
-      if (pages.length === 0) {
-        log('No content in DOCX document, creating empty page');
-        pages.push({
-          charCount: 0,
-          lineCount: 0,
-          metadata: { pageNumber: 1 },
-          pageContent: '',
-        });
+      // Handle warnings if any
+      if (result.messages.length > 0) {
+        const warnings = result.messages.filter((msg) => msg.type === 'warning');
+        if (warnings.length > 0) {
+          log('Extraction warnings:', warnings.length);
+          warnings.forEach((warning) => log('Warning:', warning.message));
+        }
       }
 
-      log('DOCX loading completed, total pages:', pages.length);
-      return pages;
+      log('DOCX loading completed');
+      return [page];
     } catch (e) {
       const error = e as Error;
       log('Error encountered while loading DOCX file');
-      console.error(`Error loading DOCX file ${filePath} using LangChain loader: ${error.message}`);
+      console.error(`Error loading DOCX file ${filePath}: ${error.message}`);
+
       const errorPage: DocumentPage = {
         charCount: 0,
         lineCount: 0,

package/packages/file-loaders/src/loaders/index.ts

@@ -1,4 +1,5 @@
 import { FileLoaderInterface, SupportedFileType } from '../types';
+import { DocLoader } from './doc';
 import { DocxLoader } from './docx';
 // import { EpubLoader } from './epub';
 import { ExcelLoader } from './excel';
@@ -10,6 +11,7 @@ import { TextLoader } from './text';
 // Key: file extension (lowercase, without leading dot) or specific type name
 // Value: Loader Class implementing FileLoaderInterface
 export const fileLoaders: Record<SupportedFileType, new () => FileLoaderInterface> = {
+  doc: DocLoader,
   docx: DocxLoader,
   // epub: EpubLoader,
   excel: ExcelLoader,

package/packages/file-loaders/src/types/word-extractor.d.ts

@@ -0,0 +1,9 @@
+declare module 'word-extractor' {
+  export default class WordExtractor {
+    extract(filePath: string): Promise<{
+      getBody: () => string;
+      getHeaders?: () => Record<string, string> | undefined;
+      text?: string;
+    }>;
+  }
+}

package/packages/file-loaders/src/types.ts

@@ -1,5 +1,5 @@
 // Define supported file types - consider using an enum or const assertion
-export type SupportedFileType = 'pdf' | 'docx' | 'txt' | 'excel' | 'pptx'; // | 'pptx' | 'latex' | 'epub' | 'code' | 'markdown';
+export type SupportedFileType = 'pdf' | 'doc' | 'docx' | 'txt' | 'excel' | 'pptx'; // | 'pptx' | 'latex' | 'epub' | 'code' | 'markdown';
 
 /**
  * 代表一个完整的已加载文件，包含文件级信息和其所有页面/块。