@nocobase/ai 2.1.0-beta.13 → 2.1.0-beta.15

package/lib/document-loader/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { Document } from '@langchain/core/documents';
+export declare const loadByWorker: (extname: string, blob: Blob) => Promise<Document[]>;

package/lib/document-loader/index.js

@@ -0,0 +1,90 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var document_loader_exports = {};
+__export(document_loader_exports, {
+  loadByWorker: () => loadByWorker
+});
+module.exports = __toCommonJS(document_loader_exports);
+var import_node_worker_threads = require("node:worker_threads");
+var import_node_path = __toESM(require("node:path"));
+const loadByWorker = /* @__PURE__ */ __name(async (extname, blob) => {
+  const buffer = Buffer.from(await blob.arrayBuffer());
+  const isTsRuntime = __filename.endsWith(".ts");
+  const workerPath = import_node_path.default.join(__dirname, `loader.worker.${isTsRuntime ? "ts" : "js"}`);
+  const worker = new import_node_worker_threads.Worker(workerPath, {
+    execArgv: isTsRuntime ? ["--require", "tsx/cjs"] : void 0
+  });
+  return new Promise((resolve, reject) => {
+    let settled = false;
+    const close = /* @__PURE__ */ __name((error, result) => {
+      if (settled) {
+        return;
+      }
+      settled = true;
+      if (error) {
+        reject(error);
+        return;
+      }
+      resolve(result || []);
+    }, "close");
+    worker.once("message", (payload) => {
+      if (payload == null ? void 0 : payload.error) {
+        close(new Error(payload.error));
+        return;
+      }
+      close(void 0, (payload == null ? void 0 : payload.documents) || []);
+    });
+    worker.once("error", (error) => close(error));
+    worker.once("exit", (code) => {
+      if (!settled && code !== 0) {
+        close(new Error(`Document loader worker exited with code ${code}`));
+      }
+    });
+    worker.postMessage({
+      extname,
+      mimeType: blob.type,
+      buffer: Uint8Array.from(buffer)
+    });
+  }).finally(() => {
+    worker.terminate().catch(() => void 0);
+  });
+}, "loadByWorker");
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  loadByWorker
+});

package/lib/document-loader/loader.worker.d.ts

@@ -0,0 +1,9 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+export {};

package/lib/document-loader/loader.worker.js

@@ -0,0 +1,83 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
+var import_pdf = require("@langchain/community/document_loaders/fs/pdf");
+var import_pptx = require("@langchain/community/document_loaders/fs/pptx");
+var import_docx = require("@langchain/community/document_loaders/fs/docx");
+var import_node_worker_threads = require("node:worker_threads");
+var import_text = require("./vendor/langchain/document_loaders/fs/text");
+var import_csv = require("@langchain/community/document_loaders/fs/csv");
+var import_xlsx = require("./xlsx");
+var _a;
+const loadPdf = /* @__PURE__ */ __name(async (blob) => {
+  const loader = new import_pdf.PDFLoader(blob);
+  return loader.load();
+}, "loadPdf");
+const loadDoc = /* @__PURE__ */ __name(async (blob, type) => {
+  const loader = new import_docx.DocxLoader(blob, { type });
+  return loader.load();
+}, "loadDoc");
+const loadPpt = /* @__PURE__ */ __name(async (blob) => {
+  const loader = new import_pptx.PPTXLoader(blob);
+  return loader.load();
+}, "loadPpt");
+const loadTxt = /* @__PURE__ */ __name(async (blob) => {
+  const loader = new import_text.TextLoader(blob);
+  return loader.load();
+}, "loadTxt");
+const loadCsv = /* @__PURE__ */ __name(async (blob) => {
+  const loader = new import_csv.CSVLoader(blob);
+  return loader.load();
+}, "loadCsv");
+const loadByExtname = /* @__PURE__ */ __name(async (payload) => {
+  const blob = new Blob([Buffer.from(payload.buffer)], { type: payload.mimeType ?? "application/octet-stream" });
+  switch (payload.extname) {
+    case ".pdf":
+      return loadPdf(blob);
+    case ".ppt":
+    case ".pptx":
+      return loadPpt(blob);
+    case ".doc":
+      return loadDoc(blob, "doc");
+    case ".docx":
+      return loadDoc(blob, "docx");
+    case ".csv":
+      return loadCsv(blob);
+    case ".xls":
+    case ".xlsx":
+      return (0, import_xlsx.loadXlsx)(blob);
+    case ".json":
+    case ".md":
+    case ".txt":
+      return loadTxt(blob);
+    default:
+      return [];
+  }
+}, "loadByExtname");
+(_a = import_node_worker_threads.parentPort) == null ? void 0 : _a.on("message", async (payload) => {
+  var _a2, _b;
+  try {
+    const documents = await loadByExtname(payload);
+    const response = {
+      documents: documents.map((doc) => ({
+        pageContent: doc.pageContent,
+        metadata: doc.metadata,
+        id: doc.id
+      }))
+    };
+    (_a2 = import_node_worker_threads.parentPort) == null ? void 0 : _a2.postMessage(response);
+  } catch (error) {
+    const response = {
+      error: String((error == null ? void 0 : error.stack) || error)
+    };
+    (_b = import_node_worker_threads.parentPort) == null ? void 0 : _b.postMessage(response);
+  }
+});

package/lib/document-loader/vendor/langchain/document_loaders/fs/text.d.ts

@@ -0,0 +1,20 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+/// <reference types="node" />
+import { Document } from '@langchain/core/documents';
+import { BaseDocumentLoader } from '@langchain/core/document_loaders/base';
+export declare class TextLoader extends BaseDocumentLoader {
+    private filePathOrBlob;
+    constructor(filePathOrBlob: any);
+    parse(raw: any): Promise<any[]>;
+    load(): Promise<Document<any>[]>;
+    static imports(): Promise<{
+        readFile: typeof import("fs/promises").readFile;
+    }>;
+}

package/lib/document-loader/vendor/langchain/document_loaders/fs/text.js

@@ -0,0 +1,99 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var text_exports = {};
+__export(text_exports, {
+  TextLoader: () => TextLoader
+});
+module.exports = __toCommonJS(text_exports);
+var import_documents = require("@langchain/core/documents");
+var import_env = require("@langchain/core/utils/env");
+var import_base = require("@langchain/core/document_loaders/base");
+const _TextLoader = class _TextLoader extends import_base.BaseDocumentLoader {
+  filePathOrBlob;
+  constructor(filePathOrBlob) {
+    super();
+    this.filePathOrBlob = filePathOrBlob;
+  }
+  async parse(raw) {
+    return [raw];
+  }
+  async load() {
+    let text;
+    let metadata;
+    if (typeof this.filePathOrBlob === "string") {
+      const { readFile } = await _TextLoader.imports();
+      text = await readFile(this.filePathOrBlob, "utf8");
+      metadata = { source: this.filePathOrBlob };
+    } else {
+      text = await this.filePathOrBlob.text();
+      metadata = { source: "blob", blobType: this.filePathOrBlob.type };
+    }
+    const parsed = await this.parse(text);
+    parsed.forEach((pageContent, i) => {
+      if (typeof pageContent !== "string") {
+        throw new Error(`Expected string, at position ${i} got ${typeof pageContent}`);
+      }
+    });
+    return parsed.map(
+      (pageContent, i) => new import_documents.Document({
+        pageContent,
+        metadata: parsed.length === 1 ? metadata : {
+          ...metadata,
+          line: i + 1
+        }
+      })
+    );
+  }
+  static async imports() {
+    try {
+      const { readFile } = await import("node:fs/promises");
+      return { readFile };
+    } catch (e) {
+      console.error(e);
+      throw new Error(
+        `Failed to load fs/promises. TextLoader available only on environment 'node'. It appears you are running environment '${(0, import_env.getEnv)()}'. See https://<link to docs> for alternatives.`
+      );
+    }
+  }
+};
+__name(_TextLoader, "TextLoader");
+let TextLoader = _TextLoader;
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  TextLoader
+});

package/lib/document-loader/xlsx.d.ts

@@ -0,0 +1,10 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { Document } from '@langchain/core/documents';
+export declare const loadXlsx: (blob: Blob) => Promise<Document[]>;

package/lib/document-loader/xlsx.js

@@ -0,0 +1,100 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var xlsx_exports = {};
+__export(xlsx_exports, {
+  loadXlsx: () => loadXlsx
+});
+module.exports = __toCommonJS(xlsx_exports);
+var import_documents = require("@langchain/core/documents");
+var XLSX = __toESM(require("xlsx"));
+const normalizeCellValue = /* @__PURE__ */ __name((value) => {
+  if (value === void 0 || value === null) {
+    return "";
+  }
+  return String(value);
+}, "normalizeCellValue");
+const trimTrailingEmptyCells = /* @__PURE__ */ __name((row) => {
+  let end = row.length;
+  while (end > 0 && normalizeCellValue(row[end - 1]).trim() === "") {
+    end -= 1;
+  }
+  return row.slice(0, end);
+}, "trimTrailingEmptyCells");
+const sheetToLines = /* @__PURE__ */ __name((sheet) => {
+  const rows = XLSX.utils.sheet_to_json(sheet, {
+    header: 1,
+    raw: false,
+    defval: "",
+    blankrows: false
+  });
+  return rows.map((row) => trimTrailingEmptyCells(Array.isArray(row) ? row : [])).filter((row) => row.length > 0).map((row) => row.map((cell) => normalizeCellValue(cell)).join("	")).filter((line) => line.trim().length > 0);
+}, "sheetToLines");
+const loadXlsx = /* @__PURE__ */ __name(async (blob) => {
+  const buffer = await blob.arrayBuffer();
+  const workbook = XLSX.read(buffer, {
+    type: "array",
+    cellText: true
+  });
+  const documents = [];
+  workbook.SheetNames.forEach((sheetName, index) => {
+    const sheet = workbook.Sheets[sheetName];
+    if (!sheet) {
+      return;
+    }
+    const lines = sheetToLines(sheet);
+    if (!lines.length) {
+      return;
+    }
+    documents.push(
+      new import_documents.Document({
+        pageContent: [`Sheet: ${sheetName}`, ...lines].join("\n"),
+        metadata: {
+          source: "blob",
+          blobType: blob.type,
+          sheetName,
+          sheetIndex: index
+        }
+      })
+    );
+  });
+  return documents;
+}, "loadXlsx");
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  loadXlsx
+});

package/lib/index.d.ts

@@ -10,3 +10,4 @@ export * from './ai-manager';
 export * from './document-manager';
 export * from './tools-manager';
 export * from './loader';
+export * from './document-loader';

package/lib/index.js

@@ -27,10 +27,12 @@ __reExport(src_exports, require("./ai-manager"), module.exports);
 __reExport(src_exports, require("./document-manager"), module.exports);
 __reExport(src_exports, require("./tools-manager"), module.exports);
 __reExport(src_exports, require("./loader"), module.exports);
+__reExport(src_exports, require("./document-loader"), module.exports);
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   ...require("./ai-manager"),
   ...require("./document-manager"),
   ...require("./tools-manager"),
-  ...require("./loader")
+  ...require("./loader"),
+  ...require("./document-loader")
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nocobase/ai",
-  "version": "2.1.0-beta.13",
+  "version": "2.1.0-beta.15",
   "description": "",
   "license": "Apache-2.0",
   "main": "./lib/index.js",
@@ -16,21 +16,23 @@
     "@langchain/langgraph-checkpoint": "^1.0.0",
     "@langchain/ollama": "^1.2.2",
     "@langchain/openai": "^1.2.7",
-    "@nocobase/logger": "2.1.0-beta.13",
-    "@nocobase/resourcer": "2.1.0-beta.13",
-    "@nocobase/utils": "2.1.0-beta.13",
+    "@nocobase/logger": "2.1.0-beta.15",
+    "@nocobase/resourcer": "2.1.0-beta.15",
+    "@nocobase/utils": "2.1.0-beta.15",
+    "d3-dsv": "2",
     "fast-glob": "^3.3.2",
     "flexsearch": "^0.8.2",
     "langchain": "^1.2.24",
     "mammoth": "^1.10.0",
     "officeparser": "^5.2.0",
     "pdf-parse": "^1.1.1",
-    "word-extractor": "^1.0.4"
+    "word-extractor": "^1.0.4",
+    "xlsx": "^0.18.5"
   },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/nocobase/nocobase.git",
     "directory": "packages/ai"
   },
-  "gitHead": "691716e5f4e5f8bd3859d65bc8a29b4e3c32209b"
+  "gitHead": "dc1aceea6357e6ab149976c2a236fc4b6bee1370"
 }

package/src/document-loader/index.ts

@@ -0,0 +1,57 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { Document } from '@langchain/core/documents';
+import { Worker } from 'node:worker_threads';
+import path from 'node:path';
+
+export const loadByWorker = async (extname: string, blob: Blob): Promise<Document[]> => {
+  const buffer = Buffer.from(await blob.arrayBuffer());
+  const isTsRuntime = __filename.endsWith('.ts');
+  const workerPath = path.join(__dirname, `loader.worker.${isTsRuntime ? 'ts' : 'js'}`);
+  const worker = new Worker(workerPath, {
+    execArgv: isTsRuntime ? ['--require', 'tsx/cjs'] : undefined,
+  });
+  return new Promise<Document[]>((resolve, reject) => {
+    let settled = false;
+    const close = (error?: Error, result?: Document[]) => {
+      if (settled) {
+        return;
+      }
+      settled = true;
+      if (error) {
+        reject(error);
+        return;
+      }
+      resolve(result || []);
+    };
+
+    worker.once('message', (payload: { documents?: Document[]; error?: string }) => {
+      if (payload?.error) {
+        close(new Error(payload.error));
+        return;
+      }
+      close(undefined, payload?.documents || []);
+    });
+    worker.once('error', (error) => close(error));
+    worker.once('exit', (code) => {
+      if (!settled && code !== 0) {
+        close(new Error(`Document loader worker exited with code ${code}`));
+      }
+    });
+
+    worker.postMessage({
+      extname,
+      mimeType: blob.type,
+      buffer: Uint8Array.from(buffer),
+    });
+  }).finally(() => {
+    worker.terminate().catch(() => undefined);
+  });
+};

package/src/document-loader/loader.worker.ts

@@ -0,0 +1,100 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { PDFLoader } from '@langchain/community/document_loaders/fs/pdf';
+import { PPTXLoader } from '@langchain/community/document_loaders/fs/pptx';
+import { DocxLoader } from '@langchain/community/document_loaders/fs/docx';
+import type { Document } from '@langchain/core/documents';
+import { parentPort } from 'node:worker_threads';
+import { TextLoader } from './vendor/langchain/document_loaders/fs/text';
+import { CSVLoader } from '@langchain/community/document_loaders/fs/csv';
+import { loadXlsx } from './xlsx';
+
+type ParsePayload = {
+  extname: string;
+  mimeType?: string;
+  buffer: Uint8Array;
+};
+
+type WorkerResponse = {
+  documents?: Array<Pick<Document, 'pageContent' | 'metadata' | 'id'>>;
+  error?: string;
+};
+
+const loadPdf = async (blob: Blob): Promise<Document[]> => {
+  const loader = new PDFLoader(blob);
+  return loader.load();
+};
+
+const loadDoc = async (blob: Blob, type: 'docx' | 'doc'): Promise<Document[]> => {
+  const loader = new DocxLoader(blob, { type });
+  return loader.load();
+};
+
+const loadPpt = async (blob: Blob): Promise<Document[]> => {
+  const loader = new PPTXLoader(blob);
+  return loader.load();
+};
+
+const loadTxt = async (blob: Blob): Promise<Document[]> => {
+  const loader = new TextLoader(blob);
+  return loader.load();
+};
+
+const loadCsv = async (blob: Blob): Promise<Document[]> => {
+  const loader = new CSVLoader(blob);
+  return loader.load();
+};
+
+const loadByExtname = async (payload: ParsePayload): Promise<Document[]> => {
+  // @ts-ignore
+  const blob = new Blob([Buffer.from(payload.buffer)], { type: payload.mimeType ?? 'application/octet-stream' });
+
+  switch (payload.extname) {
+    case '.pdf':
+      return loadPdf(blob);
+    case '.ppt':
+    case '.pptx':
+      return loadPpt(blob);
+    case '.doc':
+      return loadDoc(blob, 'doc');
+    case '.docx':
+      return loadDoc(blob, 'docx');
+    case '.csv':
+      return loadCsv(blob);
+    case '.xls':
+    case '.xlsx':
+      return loadXlsx(blob);
+    case '.json':
+    case '.md':
+    case '.txt':
+      return loadTxt(blob);
+    default:
+      return [];
+  }
+};
+
+parentPort?.on('message', async (payload: ParsePayload) => {
+  try {
+    const documents = await loadByExtname(payload);
+    const response: WorkerResponse = {
+      documents: documents.map((doc) => ({
+        pageContent: doc.pageContent,
+        metadata: doc.metadata,
+        id: doc.id,
+      })),
+    };
+    parentPort?.postMessage(response);
+  } catch (error) {
+    const response: WorkerResponse = {
+      error: String(error?.stack || error),
+    };
+    parentPort?.postMessage(response);
+  }
+});

package/src/document-loader/vendor/langchain/document_loaders/fs/text.ts

@@ -0,0 +1,72 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { Document } from '@langchain/core/documents';
+import { getEnv } from '@langchain/core/utils/env';
+import { BaseDocumentLoader } from '@langchain/core/document_loaders/base';
+
+export class TextLoader extends BaseDocumentLoader {
+  private filePathOrBlob: any;
+
+  constructor(filePathOrBlob) {
+    super();
+    this.filePathOrBlob = filePathOrBlob;
+  }
+
+  async parse(raw) {
+    return [raw];
+  }
+
+  async load() {
+    let text;
+    let metadata;
+
+    if (typeof this.filePathOrBlob === 'string') {
+      const { readFile } = await TextLoader.imports();
+      text = await readFile(this.filePathOrBlob, 'utf8');
+      metadata = { source: this.filePathOrBlob };
+    } else {
+      text = await this.filePathOrBlob.text();
+      metadata = { source: 'blob', blobType: this.filePathOrBlob.type };
+    }
+
+    const parsed = await this.parse(text);
+    parsed.forEach((pageContent, i) => {
+      if (typeof pageContent !== 'string') {
+        throw new Error(`Expected string, at position ${i} got ${typeof pageContent}`);
+      }
+    });
+
+    return parsed.map(
+      (pageContent, i) =>
+        new Document({
+          pageContent,
+          metadata:
+            parsed.length === 1
+              ? metadata
+              : {
+                  ...metadata,
+                  line: i + 1,
+                },
+        }),
+    );
+  }
+
+  static async imports() {
+    try {
+      const { readFile } = await import('node:fs/promises');
+      return { readFile };
+    } catch (e) {
+      console.error(e);
+      throw new Error(
+        `Failed to load fs/promises. TextLoader available only on environment 'node'. It appears you are running environment '${getEnv()}'. See https://<link to docs> for alternatives.`,
+      );
+    }
+  }
+}

package/src/document-loader/xlsx.ts

@@ -0,0 +1,82 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { Document } from '@langchain/core/documents';
+import * as XLSX from 'xlsx';
+
+const normalizeCellValue = (value: unknown): string => {
+  if (value === undefined || value === null) {
+    return '';
+  }
+
+  return String(value);
+};
+
+const trimTrailingEmptyCells = (row: unknown[]): unknown[] => {
+  let end = row.length;
+
+  while (end > 0 && normalizeCellValue(row[end - 1]).trim() === '') {
+    end -= 1;
+  }
+
+  return row.slice(0, end);
+};
+
+const sheetToLines = (sheet: XLSX.WorkSheet): string[] => {
+  const rows = XLSX.utils.sheet_to_json(sheet, {
+    header: 1,
+    raw: false,
+    defval: '',
+    blankrows: false,
+  }) as unknown[][];
+
+  return rows
+    .map((row) => trimTrailingEmptyCells(Array.isArray(row) ? row : []))
+    .filter((row) => row.length > 0)
+    .map((row) => row.map((cell) => normalizeCellValue(cell)).join('\t'))
+    .filter((line) => line.trim().length > 0);
+};
+
+export const loadXlsx = async (blob: Blob): Promise<Document[]> => {
+  const buffer = await blob.arrayBuffer();
+  const workbook = XLSX.read(buffer, {
+    type: 'array',
+    cellText: true,
+  });
+
+  const documents: Document[] = [];
+
+  workbook.SheetNames.forEach((sheetName, index) => {
+    const sheet = workbook.Sheets[sheetName];
+
+    if (!sheet) {
+      return;
+    }
+
+    const lines = sheetToLines(sheet);
+
+    if (!lines.length) {
+      return;
+    }
+
+    documents.push(
+      new Document({
+        pageContent: [`Sheet: ${sheetName}`, ...lines].join('\n'),
+        metadata: {
+          source: 'blob',
+          blobType: blob.type,
+          sheetName,
+          sheetIndex: index,
+        },
+      }),
+    );
+  });
+
+  return documents;
+};

package/src/index.ts

@@ -11,3 +11,4 @@ export * from './ai-manager';
 export * from './document-manager';
 export * from './tools-manager';
 export * from './loader';
+export * from './document-loader';