@nocobase/ai 2.1.23 → 2.1.25

package/lib/document-loader/index.d.ts

@@ -7,4 +7,8 @@
  * For more information, please refer to: https://www.nocobase.com/agreement.
  */
 import { Document } from '@langchain/core/documents';
-export declare const loadByWorker: (extname: string, blob: Blob) => Promise<Document[]>;
+export type DocumentLoaderWorkerOptions = {
+    filePath: string;
+    mimeType?: string;
+};
+export declare const loadByWorker: (extname: string, options: DocumentLoaderWorkerOptions) => Promise<Document[]>;

package/lib/document-loader/index.js

@@ -42,8 +42,7 @@ __export(document_loader_exports, {
 module.exports = __toCommonJS(document_loader_exports);
 var import_node_worker_threads = require("node:worker_threads");
 var import_node_path = __toESM(require("node:path"));
-const loadByWorker = /* @__PURE__ */ __name(async (extname, blob) => {
-  const buffer = Buffer.from(await blob.arrayBuffer());
+const loadByWorker = /* @__PURE__ */ __name(async (extname, options) => {
   const isTsRuntime = __filename.endsWith(".ts");
   const workerPath = import_node_path.default.join(__dirname, `loader.worker.${isTsRuntime ? "ts" : "js"}`);
   const worker = new import_node_worker_threads.Worker(workerPath, {
@@ -77,8 +76,8 @@ const loadByWorker = /* @__PURE__ */ __name(async (extname, blob) => {
     });
     worker.postMessage({
       extname,
-      mimeType: blob.type,
-      buffer: Uint8Array.from(buffer)
+      filePath: options.filePath,
+      mimeType: options.mimeType
     });
   }).finally(() => {
     worker.terminate().catch(() => void 0);

package/lib/document-loader/loader.worker.js

@@ -17,47 +17,46 @@ var import_text = require("./vendor/langchain/document_loaders/fs/text");
 var import_csv = require("@langchain/community/document_loaders/fs/csv");
 var import_xlsx = require("./xlsx");
 var _a;
-const loadPdf = /* @__PURE__ */ __name(async (blob) => {
-  const loader = new import_pdf.PDFLoader(blob);
+const loadPdf = /* @__PURE__ */ __name(async (filePath) => {
+  const loader = new import_pdf.PDFLoader(filePath);
   return loader.load();
 }, "loadPdf");
-const loadDoc = /* @__PURE__ */ __name(async (blob, type) => {
-  const loader = new import_docx.DocxLoader(blob, { type });
+const loadDoc = /* @__PURE__ */ __name(async (filePath, type) => {
+  const loader = new import_docx.DocxLoader(filePath, { type });
   return loader.load();
 }, "loadDoc");
-const loadPpt = /* @__PURE__ */ __name(async (blob) => {
-  const loader = new import_pptx.PPTXLoader(blob);
+const loadPpt = /* @__PURE__ */ __name(async (filePath) => {
+  const loader = new import_pptx.PPTXLoader(filePath);
   return loader.load();
 }, "loadPpt");
-const loadTxt = /* @__PURE__ */ __name(async (blob) => {
-  const loader = new import_text.TextLoader(blob);
+const loadTxt = /* @__PURE__ */ __name(async (filePath) => {
+  const loader = new import_text.TextLoader(filePath);
   return loader.load();
 }, "loadTxt");
-const loadCsv = /* @__PURE__ */ __name(async (blob) => {
-  const loader = new import_csv.CSVLoader(blob);
+const loadCsv = /* @__PURE__ */ __name(async (filePath) => {
+  const loader = new import_csv.CSVLoader(filePath);
   return loader.load();
 }, "loadCsv");
 const loadByExtname = /* @__PURE__ */ __name(async (payload) => {
-  const blob = new Blob([Buffer.from(payload.buffer)], { type: payload.mimeType ?? "application/octet-stream" });
   switch (payload.extname) {
     case ".pdf":
-      return loadPdf(blob);
+      return loadPdf(payload.filePath);
     case ".ppt":
     case ".pptx":
-      return loadPpt(blob);
+      return loadPpt(payload.filePath);
     case ".doc":
-      return loadDoc(blob, "doc");
+      return loadDoc(payload.filePath, "doc");
     case ".docx":
-      return loadDoc(blob, "docx");
+      return loadDoc(payload.filePath, "docx");
     case ".csv":
-      return loadCsv(blob);
+      return loadCsv(payload.filePath);
     case ".xls":
     case ".xlsx":
-      return (0, import_xlsx.loadXlsx)(blob);
+      return (0, import_xlsx.loadXlsx)(payload.filePath, payload.mimeType);
     case ".json":
     case ".md":
     case ".txt":
-      return loadTxt(blob);
+      return loadTxt(payload.filePath);
     default:
       return [];
   }

package/lib/document-loader/xlsx.d.ts

@@ -7,4 +7,4 @@
  * For more information, please refer to: https://www.nocobase.com/agreement.
  */
 import { Document } from '@langchain/core/documents';
-export declare const loadXlsx: (blob: Blob) => Promise<Document[]>;
+export declare const loadXlsx: (filePath: string, mimeType?: string) => Promise<Document[]>;

package/lib/document-loader/xlsx.js

@@ -41,6 +41,7 @@ __export(xlsx_exports, {
 });
 module.exports = __toCommonJS(xlsx_exports);
 var import_documents = require("@langchain/core/documents");
+var import_promises = require("node:fs/promises");
 var XLSX = __toESM(require("xlsx"));
 const normalizeCellValue = /* @__PURE__ */ __name((value) => {
   if (value === void 0 || value === null) {
@@ -64,10 +65,10 @@ const sheetToLines = /* @__PURE__ */ __name((sheet) => {
   });
   return rows.map((row) => trimTrailingEmptyCells(Array.isArray(row) ? row : [])).filter((row) => row.length > 0).map((row) => row.map((cell) => normalizeCellValue(cell)).join("	")).filter((line) => line.trim().length > 0);
 }, "sheetToLines");
-const loadXlsx = /* @__PURE__ */ __name(async (blob) => {
-  const buffer = await blob.arrayBuffer();
+const loadXlsx = /* @__PURE__ */ __name(async (filePath, mimeType) => {
+  const buffer = await (0, import_promises.readFile)(filePath);
   const workbook = XLSX.read(buffer, {
-    type: "array",
+    type: "buffer",
     cellText: true
   });
   const documents = [];
@@ -84,8 +85,8 @@ const loadXlsx = /* @__PURE__ */ __name(async (blob) => {
       new import_documents.Document({
         pageContent: [`Sheet: ${sheetName}`, ...lines].join("\n"),
         metadata: {
-          source: "blob",
-          blobType: blob.type,
+          source: filePath,
+          blobType: mimeType,
           sheetName,
           sheetIndex: index
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nocobase/ai",
-  "version": "2.1.23",
+  "version": "2.1.25",
   "description": "",
   "license": "Apache-2.0",
   "main": "./lib/index.js",
@@ -17,10 +17,10 @@
     "@langchain/mcp-adapters": "1.1.3",
     "@langchain/ollama": "1.2.7",
     "@langchain/openai": "1.4.7",
-    "@nocobase/data-source-manager": "2.1.23",
-    "@nocobase/logger": "2.1.23",
-    "@nocobase/resourcer": "2.1.23",
-    "@nocobase/utils": "2.1.23",
+    "@nocobase/data-source-manager": "2.1.25",
+    "@nocobase/logger": "2.1.25",
+    "@nocobase/resourcer": "2.1.25",
+    "@nocobase/utils": "2.1.25",
     "d3-dsv": "2",
     "fast-glob": "^3.3.2",
     "flexsearch": "^0.8.2",
@@ -37,5 +37,5 @@
     "url": "git+https://github.com/nocobase/nocobase.git",
     "directory": "packages/ai"
   },
-  "gitHead": "ae4b0097a6c5e4a1347431a5fde24676032ce0ce"
+  "gitHead": "f3326a2681004deee97b563fb6ab04744e26ac71"
 }

package/src/__tests__/document-loader.test.ts

@@ -0,0 +1,63 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { loadByWorker } from '../document-loader';
+import { mkdtemp, rm, writeFile } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import * as XLSX from 'xlsx';
+
+describe('Document loader worker', () => {
+  const tempDirs: string[] = [];
+
+  const createTempFile = async (filename: string, content: string | Buffer) => {
+    const tempDir = await mkdtemp(path.join(os.tmpdir(), 'nocobase-document-loader-test-'));
+    tempDirs.push(tempDir);
+    const filePath = path.join(tempDir, filename);
+    await writeFile(filePath, content);
+    return filePath;
+  };
+
+  afterEach(async () => {
+    await Promise.all(tempDirs.splice(0).map((tempDir) => rm(tempDir, { recursive: true, force: true })));
+  });
+
+  it('loads text files by path', async () => {
+    const filePath = await createTempFile('source.txt', 'hello knowledge base\nsecond line\n');
+
+    const documents = await loadByWorker('.txt', {
+      filePath,
+      mimeType: 'text/plain',
+    });
+
+    expect(documents).toHaveLength(1);
+    expect(documents[0].pageContent).toBe('hello knowledge base\nsecond line\n');
+    expect(documents[0].metadata.source).toBe(filePath);
+  });
+
+  it('loads xlsx files by path', async () => {
+    const worksheet = XLSX.utils.aoa_to_sheet([
+      ['name', 'value'],
+      ['alpha', 1],
+    ]);
+    const workbook = XLSX.utils.book_new();
+    XLSX.utils.book_append_sheet(workbook, worksheet, 'Sheet1');
+    const buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' });
+    const filePath = await createTempFile('source.xlsx', buffer);
+
+    const documents = await loadByWorker('.xlsx', {
+      filePath,
+      mimeType: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+    });
+
+    expect(documents).toHaveLength(1);
+    expect(documents[0].pageContent).toBe('Sheet: Sheet1\nname\tvalue\nalpha\t1');
+    expect(documents[0].metadata.source).toBe(filePath);
+  });
+});

package/src/document-loader/index.ts

@@ -11,8 +11,12 @@ import { Document } from '@langchain/core/documents';
 import { Worker } from 'node:worker_threads';
 import path from 'node:path';
 
-export const loadByWorker = async (extname: string, blob: Blob): Promise<Document[]> => {
-  const buffer = Buffer.from(await blob.arrayBuffer());
+export type DocumentLoaderWorkerOptions = {
+  filePath: string;
+  mimeType?: string;
+};
+
+export const loadByWorker = async (extname: string, options: DocumentLoaderWorkerOptions): Promise<Document[]> => {
   const isTsRuntime = __filename.endsWith('.ts');
   const workerPath = path.join(__dirname, `loader.worker.${isTsRuntime ? 'ts' : 'js'}`);
   const worker = new Worker(workerPath, {
@@ -48,8 +52,8 @@ export const loadByWorker = async (extname: string, blob: Blob): Promise<Documen
 
     worker.postMessage({
       extname,
-      mimeType: blob.type,
-      buffer: Uint8Array.from(buffer),
+      filePath: options.filePath,
+      mimeType: options.mimeType,
     });
   }).finally(() => {
     worker.terminate().catch(() => undefined);

package/src/document-loader/loader.worker.ts

@@ -19,7 +19,7 @@ import { loadXlsx } from './xlsx';
 type ParsePayload = {
   extname: string;
   mimeType?: string;
-  buffer: Uint8Array;
+  filePath: string;
 };
 
 type WorkerResponse = {
@@ -27,54 +27,51 @@ type WorkerResponse = {
   error?: string;
 };
 
-const loadPdf = async (blob: Blob): Promise<Document[]> => {
-  const loader = new PDFLoader(blob);
+const loadPdf = async (filePath: string): Promise<Document[]> => {
+  const loader = new PDFLoader(filePath);
   return loader.load();
 };
 
-const loadDoc = async (blob: Blob, type: 'docx' | 'doc'): Promise<Document[]> => {
-  const loader = new DocxLoader(blob, { type });
+const loadDoc = async (filePath: string, type: 'docx' | 'doc'): Promise<Document[]> => {
+  const loader = new DocxLoader(filePath, { type });
   return loader.load();
 };
 
-const loadPpt = async (blob: Blob): Promise<Document[]> => {
-  const loader = new PPTXLoader(blob);
+const loadPpt = async (filePath: string): Promise<Document[]> => {
+  const loader = new PPTXLoader(filePath);
   return loader.load();
 };
 
-const loadTxt = async (blob: Blob): Promise<Document[]> => {
-  const loader = new TextLoader(blob);
+const loadTxt = async (filePath: string): Promise<Document[]> => {
+  const loader = new TextLoader(filePath);
   return loader.load();
 };
 
-const loadCsv = async (blob: Blob): Promise<Document[]> => {
-  const loader = new CSVLoader(blob);
+const loadCsv = async (filePath: string): Promise<Document[]> => {
+  const loader = new CSVLoader(filePath);
   return loader.load();
 };
 
 const loadByExtname = async (payload: ParsePayload): Promise<Document[]> => {
-  // @ts-ignore
-  const blob = new Blob([Buffer.from(payload.buffer)], { type: payload.mimeType ?? 'application/octet-stream' });
-
   switch (payload.extname) {
     case '.pdf':
-      return loadPdf(blob);
+      return loadPdf(payload.filePath);
     case '.ppt':
     case '.pptx':
-      return loadPpt(blob);
+      return loadPpt(payload.filePath);
     case '.doc':
-      return loadDoc(blob, 'doc');
+      return loadDoc(payload.filePath, 'doc');
     case '.docx':
-      return loadDoc(blob, 'docx');
+      return loadDoc(payload.filePath, 'docx');
     case '.csv':
-      return loadCsv(blob);
+      return loadCsv(payload.filePath);
     case '.xls':
     case '.xlsx':
-      return loadXlsx(blob);
+      return loadXlsx(payload.filePath, payload.mimeType);
     case '.json':
     case '.md':
     case '.txt':
-      return loadTxt(blob);
+      return loadTxt(payload.filePath);
     default:
       return [];
   }

package/src/document-loader/xlsx.ts

@@ -8,6 +8,7 @@
  */
 
 import { Document } from '@langchain/core/documents';
+import { readFile } from 'node:fs/promises';
 import * as XLSX from 'xlsx';
 
 const normalizeCellValue = (value: unknown): string => {
@@ -43,10 +44,10 @@ const sheetToLines = (sheet: XLSX.WorkSheet): string[] => {
     .filter((line) => line.trim().length > 0);
 };
 
-export const loadXlsx = async (blob: Blob): Promise<Document[]> => {
-  const buffer = await blob.arrayBuffer();
+export const loadXlsx = async (filePath: string, mimeType?: string): Promise<Document[]> => {
+  const buffer = await readFile(filePath);
   const workbook = XLSX.read(buffer, {
-    type: 'array',
+    type: 'buffer',
     cellText: true,
   });
 
@@ -69,8 +70,8 @@ export const loadXlsx = async (blob: Blob): Promise<Document[]> => {
       new Document({
         pageContent: [`Sheet: ${sheetName}`, ...lines].join('\n'),
         metadata: {
-          source: 'blob',
-          blobType: blob.type,
+          source: filePath,
+          blobType: mimeType,
           sheetName,
           sheetIndex: index,
         },