@nocobase/plugin-ai 2.1.0-beta.32 → 2.1.0-beta.34

package/dist/server/document-loader/cached.js

@@ -43,24 +43,21 @@ var import_promises = __toESM(require("node:fs/promises"));
 var import_node_os = __toESM(require("node:os"));
 var import_node_path = __toESM(require("node:path"));
 var import_documents = require("@langchain/core/documents");
-var import_constants = require("./constants");
 var import_utils = require("./utils");
 class CachedDocumentLoader {
   constructor(plugin, options) {
     this.plugin = plugin;
     this.options = options;
   }
-  async load(file) {
+  _cache = null;
+  async load(file, options) {
     const sourceFile = this.toPlainObject(file);
     if (!this.options.supports(sourceFile)) {
       return {
         supported: false,
         fromCache: false,
         text: "",
-        documents: [],
-        meta: {
-          sourceFileId: sourceFile.id
-        }
+        documents: []
       };
     }
     if (sourceFile.size === 0) {
@@ -68,132 +65,60 @@ class CachedDocumentLoader {
         supported: true,
         fromCache: false,
         text: "",
-        documents: [],
-        meta: {
-          sourceFileId: sourceFile.id
-        }
+        documents: []
       };
     }
     const cached = await this.loadFromCache(sourceFile);
     if (cached) {
       return cached;
     }
-    try {
-      const documents = await this.options.loader.load(sourceFile);
-      const text = this.documentsToText(documents);
-      const parsedFile = await this.persistParsedText(sourceFile, text);
-      await this.updateSourceMeta(sourceFile, {
-        status: "ready",
-        parserVersion: this.options.parserVersion,
-        parsedFileId: parsedFile.id,
-        parsedFilename: parsedFile.filename,
-        parsedMimetype: this.options.parsedMimetype,
-        updatedAt: (/* @__PURE__ */ new Date()).toISOString()
-      });
-      return {
-        supported: true,
-        fromCache: false,
-        text,
-        documents,
-        meta: {
-          sourceFileId: sourceFile.id,
-          parsedFileId: parsedFile.id
-        }
-      };
-    } catch (error) {
-      await this.updateSourceMeta(sourceFile, {
-        status: "failed",
-        parserVersion: this.options.parserVersion,
-        updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
-        error: (error == null ? void 0 : error.message) ?? String(error)
-      });
-      throw error;
-    }
+    const documents = await this.options.loader.load(sourceFile, options);
+    const text = this.documentsToText(documents);
+    await this.persistParsedText(sourceFile, text);
+    return {
+      supported: true,
+      fromCache: false,
+      text,
+      documents
+    };
   }
   async loadFromCache(sourceFile) {
-    const meta = this.getParseMeta(sourceFile.meta);
-    if (!meta || meta.status !== "ready" || meta.parserVersion !== this.options.parserVersion || !meta.parsedFileId) {
+    const cacheKey = this.getCacheKey(sourceFile);
+    if (!cacheKey) {
       return null;
     }
-    const parsedModel = await this.aiFilesRepo.findById(meta.parsedFileId);
-    if (!parsedModel) {
+    const cache = await this.getCache();
+    const filePath = await cache.get(cacheKey);
+    if (!filePath) {
+      return null;
+    }
+    try {
+      const stat = await import_promises.default.stat(filePath);
+      if (!stat.isFile()) {
+        return null;
+      }
+    } catch {
       return null;
     }
-    const parsedFile = this.toPlainObject(parsedModel);
-    const text = await this.readTextFile(parsedFile);
+    const text = await import_promises.default.readFile(filePath, "utf-8");
     const extname = (0, import_utils.resolveExtname)(sourceFile);
     const documents = this.toDocumentsFromText(text, sourceFile, extname);
     return {
       supported: true,
       fromCache: true,
       text,
-      documents,
-      meta: {
-        sourceFileId: sourceFile.id,
-        parsedFileId: parsedFile.id
-      }
+      documents
     };
   }
   async persistParsedText(sourceFile, text) {
-    const tempFilePath = import_node_path.default.join(
-      import_node_os.default.tmpdir(),
-      `${sourceFile.id ?? Date.now()}.${Date.now()}.parsed.${this.options.parsedFileExtname}`
-    );
-    await import_promises.default.writeFile(tempFilePath, text, "utf-8");
-    try {
-      const storageName = await this.resolveStorageName(sourceFile);
-      const created = await this.fileManager.createFileRecord({
-        collectionName: "aiFiles",
-        filePath: tempFilePath,
-        storageName,
-        values: {
-          title: `${sourceFile.title ?? sourceFile.filename ?? "document"} (parsed)`,
-          mimetype: this.options.parsedMimetype,
-          meta: {
-            parserVersion: this.options.parserVersion,
-            sourceFileId: sourceFile.id
-          }
-        }
-      });
-      return this.toPlainObject(created);
-    } finally {
-      await import_promises.default.rm(tempFilePath, { force: true });
-    }
-  }
-  async updateSourceMeta(sourceFile, documentParse) {
-    if (!sourceFile.id) {
-      return;
-    }
-    const nextMeta = {
-      ...sourceFile.meta ?? {},
-      [import_constants.DOCUMENT_PARSE_META_KEY]: documentParse
-    };
-    await this.aiFilesRepo.update({
-      filter: {
-        id: sourceFile.id
-      },
-      values: {
-        meta: nextMeta
-      }
-    });
-  }
-  async resolveStorageName(file) {
-    if (!file.storageId) {
-      return void 0;
-    }
-    if (!this.fileManager.storagesCache.size) {
-      await this.fileManager.loadStorages();
-    }
-    const storage = this.fileManager.storagesCache.get(file.storageId);
-    return storage == null ? void 0 : storage.name;
-  }
-  async readTextFile(file) {
-    const { stream } = await this.fileManager.getFileStream(file);
-    const chunks = [];
-    for await (const chunk of stream) {
-      chunks.push(chunk);
+    const cacheKey = this.getCacheKey(sourceFile);
+    if (!cacheKey) {
+      return null;
     }
-    return Buffer.concat(chunks).toString("utf-8");
+    const tempFilePath = import_node_path.default.join(import_node_os.default.tmpdir(), `${cacheKey}.${Date.now()}.parsed.${this.options.parsedFileExtname}`);
+    await import_promises.default.writeFile(tempFilePath, text, "utf-8");
+    const cache = await this.getCache();
+    await cache.set(cacheKey, tempFilePath, 30 * 60 * 1e3);
   }
   documentsToText(documents) {
     return documents.map((doc) => doc.pageContent).join("\n\n");
@@ -212,23 +137,27 @@ class CachedDocumentLoader {
       })
     ];
   }
-  getParseMeta(meta) {
-    if (!meta || typeof meta !== "object") {
-      return null;
-    }
-    return meta[import_constants.DOCUMENT_PARSE_META_KEY] ?? null;
-  }
   toPlainObject(file) {
     if (file == null ? void 0 : file.toJSON) {
       return file.toJSON();
     }
     return file;
   }
-  get aiFilesRepo() {
-    return this.plugin.db.getRepository("aiFiles");
+  async getCache() {
+    this._cache ??= await this.plugin.app.cacheManager.createCache({
+      name: "ai-employee:document-loader:parsed",
+      store: "memory"
+    });
+    return this._cache;
   }
-  get fileManager() {
-    return this.plugin.app.pm.get("file-manager");
+  getCacheKey(sourceFile) {
+    if (!sourceFile) {
+      return null;
+    }
+    if (!sourceFile.id || !sourceFile.storageId) {
+      return null;
+    }
+    return `${sourceFile.id}@${sourceFile.storageId}`;
   }
 }
 // Annotate the CommonJS export names for ESM import in node:

package/dist/server/document-loader/loader.d.ts

@@ -12,6 +12,6 @@ import { ParseableFile } from './types';
 export declare class DocumentLoader {
     private readonly fileManager;
     constructor(fileManager: PluginFileManagerServer);
-    load(file: ParseableFile): Promise<Document[]>;
+    load(file: ParseableFile, options?: any): Promise<Document[]>;
     private streamToBlob;
 }

package/dist/server/document-loader/loader.js

@@ -36,12 +36,12 @@ class DocumentLoader {
   constructor(fileManager) {
     this.fileManager = fileManager;
   }
-  async load(file) {
+  async load(file, options) {
     const extname = (0, import_utils.resolveExtname)(file);
     if (!import_constants.SUPPORTED_DOCUMENT_EXTNAMES.includes(extname)) {
       return [];
     }
-    const { stream, contentType } = await this.fileManager.getFileStream(file);
+    const { stream, contentType } = await this.fileManager.getFileStream(file, options);
     const blob = await this.streamToBlob(stream, contentType ?? file.mimetype);
     return await (0, import_ai.loadByWorker)(extname, blob);
   }

package/dist/server/document-loader/types.d.ts

@@ -21,11 +21,6 @@ export type ParsedDocumentResult = {
     fromCache: boolean;
     text: string;
     documents: Document[];
-    meta?: {
-        sourceFileId?: string | number;
-        parsedFileId?: string | number;
-        extname?: string;
-    };
 };
 export type ParseableFile = {
     id?: number | string;
@@ -38,5 +33,5 @@ export type ParseableFile = {
     size?: number;
 };
 export type DocumentLoaderLike = {
-    load(file: ParseableFile): Promise<Document[]>;
+    load(file: ParseableFile, options?: any): Promise<Document[]>;
 };

package/dist/server/features/vector-database-provider.d.ts

@@ -14,6 +14,10 @@ export interface VectorDatabaseProviderFeature {
         success: boolean;
         error?: string;
     }>;
+    beforeCreate<T>(providerName: string, connectParams: T, options?: any): Promise<{
+        status: number;
+        message?: string;
+    }>;
     createVectorStore<T, R>(providerName: string, embeddings: EmbeddingsInterface, connectParams: T): Promise<R>;
     listProviders(): VectorDatabaseProviderInfo<unknown, unknown>[];
 }
@@ -28,5 +32,9 @@ export type VectorDatabaseProvider<T, R> = {
         success: boolean;
         error?: string;
     }>;
+    beforeCreate(connectParams: T, options?: any): Promise<{
+        status: number;
+        message?: string;
+    }>;
     createVectorStore(embeddings: EmbeddingsInterface, connectParams: T): Promise<R>;
 };

package/dist/server/llm-providers/kimi/document-loader.d.ts

@@ -17,7 +17,7 @@ export declare class KimiDocumentLoader {
         apiKey?: string;
         baseURL?: string;
     });
-    load(file: ParseableFile): Promise<Document[]>;
+    load(file: ParseableFile, options?: any): Promise<Document[]>;
     private parseByApi;
     private deleteRemoteFile;
     private formatApiError;

package/dist/server/llm-providers/kimi/document-loader.js

@@ -54,8 +54,8 @@ class KimiDocumentLoader {
     this.client = this.createClient();
   }
   client;
-  async load(file) {
-    const text = await this.parseByApi(file);
+  async load(file, options) {
+    const text = await this.parseByApi(file, options);
     if (!text) {
       return [];
     }
@@ -69,12 +69,12 @@ class KimiDocumentLoader {
       })
     ];
   }
-  async parseByApi(sourceFile) {
+  async parseByApi(sourceFile, options) {
     let uploadedFileId = "";
     const safeFilename = import_node_path.default.basename(sourceFile.filename || "document");
     const tempFilePath = import_node_path.default.join(import_node_os.default.tmpdir(), `${sourceFile.id ?? Date.now()}.${Date.now()}.${safeFilename}`);
     try {
-      const { stream } = await this.fileManager.getFileStream(sourceFile);
+      const { stream } = await this.fileManager.getFileStream(sourceFile, options);
       await (0, import_promises2.pipeline)(stream, (0, import_node_fs.createWriteStream)(tempFilePath));
       let uploaded;
       try {

package/dist/server/llm-providers/provider.d.ts

@@ -70,7 +70,7 @@ export declare abstract class LLMProvider {
     protected isApiSupportedAttachment(attachment: AttachmentModel): boolean;
     protected isDocumentLoaderSupportedAttachment(attachment: AttachmentModel): boolean;
     protected convertToContent(ctx: Context, attachment: any): Promise<ParsedAttachmentResult>;
-    protected loadDocument(_ctx: Context, attachment: any): Promise<any>;
+    protected loadDocument(ctx: Context, attachment: any): Promise<any>;
     getStructuredOutputOptions(structuredOutput: AIChatContext['structuredOutput']): any;
     testFlight(): Promise<{
         status: 'success' | 'error';

package/dist/server/llm-providers/provider.js

@@ -202,9 +202,18 @@ class LLMProvider {
       };
     }
   }
-  async loadDocument(_ctx, attachment) {
+  async loadDocument(ctx, attachment) {
+    const referer = ctx.get("referer") || "";
+    const ua = ctx.get("user-agent") || "";
     const safeFilename = attachment.filename ? import_node_path.default.basename(attachment.filename) : "document";
-    const parsed = await this.documentLoader.load(attachment);
+    const parsed = await this.documentLoader.load(attachment, {
+      requestOptions: {
+        headers: {
+          Referer: referer,
+          "User-Agent": ua
+        }
+      }
+    });
     if (!parsed.supported) {
       return {
         placement: "system",

package/dist/server/manager/llm-stream-manager.d.ts

@@ -7,6 +7,11 @@
  * For more information, please refer to: https://www.nocobase.com/agreement.
  */
 import PluginAIServer from '../plugin';
+type LLMStreamOptions = {
+    pollInterval?: number;
+    initialWaitTimeout?: number;
+    signal?: AbortSignal;
+};
 export declare class LLMStreamCachedManager {
     private readonly plugin;
     private cachePromise?;
@@ -14,14 +19,10 @@ export declare class LLMStreamCachedManager {
     getCached(sessionId: string): LLMStreamCached;
     clear(sessionId: string): Promise<void>;
     append(sessionId: string, chunk: string): Promise<void>;
-    stream(sessionId: string, options?: {
-        pollInterval?: number;
-        initialWaitTimeout?: number;
-    }): AsyncGenerator<string, void, void>;
+    stream(sessionId: string, options?: LLMStreamOptions): AsyncGenerator<string, void, void>;
     private getChunks;
     private getCache;
-    private withLock;
-    private getLockKey;
+    private get store();
 }
 export declare class LLMStreamCached {
     private readonly sessionId;
@@ -30,8 +31,13 @@ export declare class LLMStreamCached {
     clear(): Promise<void>;
     append(chunk: string): Promise<void>;
     skipped(): Promise<void>;
-    stream(options?: {
-        pollInterval?: number;
-        initialWaitTimeout?: number;
-    }): AsyncGenerator<string, void, void>;
+    stream(options?: LLMStreamOptions): AsyncGenerator<string, void, void>;
 }
+export declare class BufferedLLMStreamCached extends LLMStreamCached {
+    private buffer;
+    private interval;
+    clear(): Promise<void>;
+    append(chunk: string): Promise<void>;
+    private flush;
+}
+export {};

package/dist/server/manager/llm-stream-manager.js

@@ -26,54 +26,59 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var llm_stream_manager_exports = {};
 __export(llm_stream_manager_exports, {
+  BufferedLLMStreamCached: () => BufferedLLMStreamCached,
   LLMStreamCached: () => LLMStreamCached,
   LLMStreamCachedManager: () => LLMStreamCachedManager
 });
 module.exports = __toCommonJS(llm_stream_manager_exports);
-var import_utils = require("@nocobase/utils");
 const CACHE_NAME = "ai-llm-stream-cache";
-const LOCK_KEY_PREFIX = "ai-llm-stream-lock";
-const WRITE_LOCK_TTL = 3e3;
 const CACHE_TTL = 10 * 60 * 1e3;
 const STREAM_END_MARK = '"type":"stream_end"';
 const SKIPPED_MARK = "__skipped__";
-const DEFAULT_POLL_INTERVAL = 50;
-const DEFAULT_INITIAL_WAIT_TIMEOUT = 1e3;
+const DEFAULT_INITIAL_WAIT_TIMEOUT = 1e4;
 class LLMStreamCachedManager {
   constructor(plugin) {
     this.plugin = plugin;
   }
   cachePromise;
   getCached(sessionId) {
+    if (this.store !== "memory") {
+      return new BufferedLLMStreamCached(sessionId, this);
+    }
     return new LLMStreamCached(sessionId, this);
   }
   async clear(sessionId) {
-    await this.withLock(sessionId, async () => {
-      const cache = await this.getCache();
-      await cache.del(sessionId);
-    });
+    const cache = await this.getCache();
+    await cache.del(sessionId);
   }
   async append(sessionId, chunk) {
-    await this.withLock(sessionId, async () => {
-      const cache = await this.getCache();
-      const chunks = await cache.get(sessionId) ?? [];
-      chunks.push(chunk);
-      await cache.set(sessionId, chunks, CACHE_TTL);
-    });
+    const cache = await this.getCache();
+    const chunks = await cache.get(sessionId) ?? [];
+    chunks.push(chunk);
+    await cache.set(sessionId, chunks, CACHE_TTL);
   }
   async *stream(sessionId, options) {
-    const pollInterval = (options == null ? void 0 : options.pollInterval) ?? DEFAULT_POLL_INTERVAL;
+    const pollInterval = (options == null ? void 0 : options.pollInterval) ?? (this.store !== "memory" ? 1e3 : 100);
     const initialWaitTimeout = (options == null ? void 0 : options.initialWaitTimeout) ?? DEFAULT_INITIAL_WAIT_TIMEOUT;
+    const signal = options == null ? void 0 : options.signal;
     let offset = 0;
     let waited = 0;
     let completed = false;
     while (!completed) {
+      if (signal == null ? void 0 : signal.aborted) {
+        return;
+      }
       const chunks = await this.getChunks(sessionId);
       const lastSkippedIndex = chunks.lastIndexOf(SKIPPED_MARK);
+      let hasNewChunks = false;
       if (lastSkippedIndex >= offset) {
         offset = lastSkippedIndex + 1;
       }
       while (offset < chunks.length) {
+        if (signal == null ? void 0 : signal.aborted) {
+          return;
+        }
+        hasNewChunks = true;
         const chunk = chunks[offset++];
         yield chunk;
         waited = 0;
@@ -85,16 +90,15 @@ class LLMStreamCachedManager {
       if (completed) {
         return;
       }
-      if (!chunks.length) {
-        if (offset > 0) {
-          return;
-        }
+      if (!hasNewChunks) {
         if (waited >= initialWaitTimeout) {
           return;
         }
         waited += pollInterval;
       }
-      await (0, import_utils.sleep)(pollInterval);
+      if (await abortableSleep(pollInterval, signal)) {
+        return;
+      }
     }
   }
   async getChunks(sessionId) {
@@ -104,15 +108,13 @@ class LLMStreamCachedManager {
   async getCache() {
     this.cachePromise ??= this.plugin.app.cacheManager.createCache({
       name: CACHE_NAME,
-      store: this.plugin.app.cacheManager.defaultStore
+      store: this.store
     });
     return this.cachePromise;
   }
-  async withLock(sessionId, fn) {
-    return await this.plugin.app.lockManager.runExclusive(this.getLockKey(sessionId), fn, WRITE_LOCK_TTL);
-  }
-  getLockKey(sessionId) {
-    return `${LOCK_KEY_PREFIX}:${sessionId}`;
+  get store() {
+    var _a;
+    return ((_a = this.plugin.app.options.cacheManager) == null ? void 0 : _a.defaultStore) ?? "memory";
   }
 }
 class LLMStreamCached {
@@ -135,8 +137,100 @@ class LLMStreamCached {
     }
   }
 }
+class BufferedLLMStreamCached extends LLMStreamCached {
+  buffer = new ChunksBuffer();
+  interval;
+  async clear() {
+    clearInterval(this.interval);
+    delete this.interval;
+    await this.flush();
+    await super.clear();
+  }
+  async append(chunk) {
+    if (!this.interval) {
+      this.interval = setInterval(() => {
+        void this.flush();
+      }, 1e3);
+    }
+    this.buffer.append(chunk);
+  }
+  async flush() {
+    if (this.buffer.isEmpty()) {
+      return;
+    }
+    const chunks = this.buffer.compact();
+    this.buffer.clear();
+    const appendChunks = async () => {
+      for (const chunk of chunks) {
+        await super.append(chunk);
+      }
+    };
+    await appendChunks();
+  }
+}
+class ChunksBuffer {
+  _size = 0;
+  _chunks = [];
+  append(chunk) {
+    this._size += chunk.length;
+    this._chunks.push(chunk);
+  }
+  clear() {
+    this._size = 0;
+    this._chunks = [];
+  }
+  isEmpty() {
+    return this._size === 0;
+  }
+  get size() {
+    return this._size;
+  }
+  get chunks() {
+    return this._chunks;
+  }
+  compact() {
+    const chunks = [];
+    let buffer = "";
+    for (const chunk of this._chunks) {
+      if (chunk === SKIPPED_MARK) {
+        if (buffer) {
+          chunks.push(buffer);
+          buffer = "";
+        }
+        chunks.push(SKIPPED_MARK);
+      } else {
+        buffer += chunk;
+      }
+    }
+    if (buffer) {
+      chunks.push(buffer);
+    }
+    return chunks;
+  }
+}
+function abortableSleep(ms, signal) {
+  if (signal == null ? void 0 : signal.aborted) {
+    return Promise.resolve(true);
+  }
+  return new Promise((resolve) => {
+    const timeout = setTimeout(() => {
+      cleanup();
+      resolve(false);
+    }, ms);
+    const cleanup = () => {
+      clearTimeout(timeout);
+      signal == null ? void 0 : signal.removeEventListener("abort", abort);
+    };
+    const abort = () => {
+      cleanup();
+      resolve(true);
+    };
+    signal == null ? void 0 : signal.addEventListener("abort", abort, { once: true });
+  });
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  BufferedLLMStreamCached,
   LLMStreamCached,
   LLMStreamCachedManager
 });

package/dist/server/migrations/20260407170416-ai-employee-knowledge-base-add-key.js

@@ -33,7 +33,7 @@ var import_server = require("@nocobase/server");
 class ai_employee_knowledge_base_add_key_default extends import_server.Migration {
   on = "afterSync";
   // 'beforeLoad' or 'afterLoad'
-  appVersion = "<2.1.0";
+  appVersion = "<2.2.0";
   async up() {
     var _a, _b, _c, _d;
     const aiEmployeesRepo = this.app.db.getRepository("aiEmployees");

package/dist/server/resource/aiConversations.d.ts

@@ -7,28 +7,27 @@
  * For more information, please refer to: https://www.nocobase.com/agreement.
  */
 import { Context, Next } from '@nocobase/actions';
-declare function parallelConversationsLimit(ctx: Context, next: Next): Promise<never>;
+declare function loginInCheck(ctx: Context, next: Next): Promise<never>;
 declare const _default: {
     name: string;
     middlewares: {
-        only: string[];
-        handler: typeof parallelConversationsLimit;
+        handler: typeof loginInCheck;
     }[];
     actions: {
         list(ctx: Context, next: Next): Promise<void>;
-        unreadCount(ctx: Context, next: Next): Promise<never>;
-        unreadCounts(ctx: Context, next: Next): Promise<never>;
-        create(ctx: Context, next: Next): Promise<never>;
-        update(ctx: Context, next: Next): Promise<never>;
+        unreadCount(ctx: Context, next: Next): Promise<void>;
+        unreadCounts(ctx: Context, next: Next): Promise<void>;
+        create(ctx: Context, next: Next): Promise<void>;
+        update(ctx: Context, next: Next): Promise<void>;
         updateOptions(ctx: Context, next: Next): Promise<never>;
         destroy(ctx: Context, next: Next): Promise<void>;
-        getMessages(ctx: Context, next: Next): Promise<never>;
+        getMessages(ctx: Context, next: Next): Promise<void>;
         updateToolArgs(ctx: Context, next: Next): Promise<any>;
-        sendMessages(ctx: Context, next: Next): Promise<any>;
-        abort(ctx: Context, next: Next): Promise<never>;
-        resumeStream(ctx: Context, next: Next): Promise<never>;
-        resendMessages(ctx: Context, next: Next): Promise<any>;
-        updateUserDecision(ctx: Context, next: Next): Promise<never>;
+        sendMessages(ctx: Context, next: Next): Promise<void>;
+        abort(ctx: Context, next: Next): Promise<void>;
+        resumeStream(ctx: Context, next: Next): Promise<void>;
+        resendMessages(ctx: Context, next: Next): Promise<void>;
+        updateUserDecision(ctx: Context, next: Next): Promise<void>;
         resumeToolCall(ctx: Context, next: Next): Promise<any>;
     };
 };