@nocobase/plugin-ai 2.1.0-beta.29 → 2.1.0-beta.30

package/dist/server/ai-employees/ai-employee.js

@@ -70,6 +70,7 @@ class AIEmployee {
   legacy;
   tools;
   inWorkflow;
+  streamCached;
   constructor({
     ctx,
     employee,
@@ -94,10 +95,17 @@ class AIEmployee {
     this.legacy = legacy;
     this.from = from;
     this.tools = tools;
+    this.streamCached = this.plugin.llmStreamCachedManager.getCached(sessionId);
     const builtInManager = this.plugin.builtInManager;
     builtInManager.setupBuiltInInfo(ctx, this.employee);
     this.webSearch = webSearch;
-    this.protocol = ChatStreamProtocol.fromContext(ctx);
+    this.protocol = ChatStreamProtocol.fromContext(ctx, async (chunk) => {
+      try {
+        await this.streamCached.append(chunk);
+      } catch (error) {
+        this.logger.warn("Failed to append LLM stream cache", { sessionId: this.sessionId, error });
+      }
+    });
   }
   async getFormatMessages(userMessages) {
     const { provider } = await this.plugin.aiManager.getLLMService({
@@ -194,6 +202,7 @@ class AIEmployee {
     userMessages = [],
     userDecisions
   }) {
+    await this.streamCached.clear();
     await this.aiConversationsRepo.update({
       values: { llmActiveState: "streaming" },
       filter: {
@@ -228,11 +237,12 @@ class AIEmployee {
       return false;
     } finally {
       await this.aiConversationsRepo.update({
-        values: { llmActiveState: "idle" },
+        values: { llmActiveState: "idle", read: false },
         filter: {
           sessionId: this.sessionId
         }
       });
+      await this.streamCached.clear();
     }
   }
   async invoke({
@@ -386,6 +396,14 @@ class AIEmployee {
         }
       } catch (e) {
         this.logger.error("Fail to save message after conversation abort", gathered);
+      } finally {
+        await this.aiConversationsRepo.update({
+          values: { llmActiveState: "idle", read: true },
+          filter: {
+            sessionId: this.sessionId
+          }
+        });
+        await this.streamCached.clear();
       }
     });
     try {
@@ -394,7 +412,7 @@ class AIEmployee {
         from: this.from,
         username: this.employee.username
       };
-      this.protocol.with(aiEmployeeConversation).startStream();
+      await this.protocol.with(aiEmployeeConversation).startStream();
       for await (const [mode, chunks] of stream) {
         if (mode === "messages") {
           const [chunk, metadata] = chunks;
@@ -404,24 +422,24 @@ class AIEmployee {
             if (chunk.content) {
               if (isReasoning) {
                 isReasoning = false;
-                this.protocol.with(currentConversation).stopReasoning();
+                await this.protocol.with(currentConversation).stopReasoning();
               }
               const parsedContent = provider.parseResponseChunk(chunk.content);
               if (parsedContent) {
-                this.protocol.with(currentConversation).content(parsedContent);
+                await this.protocol.with(currentConversation).content(parsedContent);
               }
             }
             if ((_a = chunk.tool_call_chunks) == null ? void 0 : _a.length) {
-              this.protocol.with(currentConversation).toolCallChunks(chunk.tool_call_chunks);
+              await this.protocol.with(currentConversation).toolCallChunks(chunk.tool_call_chunks);
             }
             const webSearch = provider.parseWebSearchAction(chunk);
             if (webSearch == null ? void 0 : webSearch.length) {
-              this.protocol.with(currentConversation).webSearch(webSearch);
+              await this.protocol.with(currentConversation).webSearch(webSearch);
             }
             const reasoningContent = provider.parseReasoningContent(chunk);
             if (reasoningContent) {
               isReasoning = true;
-              this.protocol.with(currentConversation).reasoning(reasoningContent);
+              await this.protocol.with(currentConversation).reasoning(reasoningContent);
             }
           }
         } else if (mode === "updates") {
@@ -431,8 +449,8 @@ class AIEmployee {
             await this.handleInterruptedToolCalls(
               interrupt,
               (sessionId) => aiMessageIdMap.get(sessionId),
-              ({ messageId, interruptAction, toolCall, currentConversation }) => {
-                this.protocol.with(currentConversation).toolCallStatus({
+              async ({ messageId, interruptAction, toolCall, currentConversation }) => {
+                await this.protocol.with(currentConversation).toolCallStatus({
                   toolCall: {
                     messageId,
                     id: toolCall.id,
@@ -448,6 +466,7 @@ class AIEmployee {
         } else if (mode === "custom") {
           const { currentConversation } = chunks;
           if (chunks.action === "AfterAIMessageSaved") {
+            await this.streamCached.skipped();
             aiMessageIdMap.set(currentConversation.sessionId, chunks.body.messageId);
             const data = responseMetadata.get(chunks.body.id);
             if (data) {
@@ -476,11 +495,11 @@ class AIEmployee {
               }
             }
           } else if (chunks.action === "initToolCalls") {
-            this.protocol.with(currentConversation).toolCalls(chunks.body);
+            await this.protocol.with(currentConversation).toolCalls(chunks.body);
           } else if (chunks.action === "beforeToolCall") {
             const toolsMap = await this.getToolsMap();
             const willInterrupt = this.shouldInterruptToolCall(toolsMap.get((_d = (_c = chunks.body) == null ? void 0 : _c.toolCall) == null ? void 0 : _d.name));
-            this.protocol.with(currentConversation).toolCallStatus({
+            await this.protocol.with(currentConversation).toolCallStatus({
               toolCall: {
                 messageId: (_f = (_e = chunks.body) == null ? void 0 : _e.toolCall) == null ? void 0 : _f.messageId,
                 id: (_h = (_g = chunks.body) == null ? void 0 : _g.toolCall) == null ? void 0 : _h.id,
@@ -492,7 +511,7 @@ class AIEmployee {
           } else if (chunks.action === "afterToolCall") {
             const toolsMap = await this.getToolsMap();
             const willInterrupt = this.shouldInterruptToolCall(toolsMap.get((_l = (_k = chunks.body) == null ? void 0 : _k.toolCall) == null ? void 0 : _l.name));
-            this.protocol.with(currentConversation).toolCallStatus({
+            await this.protocol.with(currentConversation).toolCallStatus({
               toolCall: {
                 messageId: (_n = (_m = chunks.body) == null ? void 0 : _m.toolCall) == null ? void 0 : _n.messageId,
                 id: (_p = (_o = chunks.body) == null ? void 0 : _o.toolCall) == null ? void 0 : _p.id,
@@ -516,7 +535,7 @@ class AIEmployee {
               for (const { metadata } of messages) {
                 const tools = toolsMap.get(metadata.toolName);
                 const toolCallResult = toolCallResultMap.get(metadata.toolCallId);
-                this.protocol.with(currentConversation).toolCallStatus({
+                await this.protocol.with(currentConversation).toolCallStatus({
                   toolCall: {
                     messageId,
                     id: metadata.toolCallId,
@@ -531,9 +550,9 @@ class AIEmployee {
                 });
               }
             }
-            this.protocol.with(currentConversation).newMessage();
+            await this.protocol.with(currentConversation).newMessage();
           } else if (chunks.action === "afterSubAgentInvoke") {
-            this.protocol.with(currentConversation).subAgentCompleted();
+            await this.protocol.with(currentConversation).subAgentCompleted();
           }
         }
       }
@@ -541,7 +560,7 @@ class AIEmployee {
         this.sendErrorResponse("Empty message");
         return;
       }
-      this.protocol.with(aiEmployeeConversation).endStream();
+      await this.protocol.with(aiEmployeeConversation).endStream();
     } catch (err) {
       this.ctx.log.error(err);
       if (err.name === "GraphRecursionError") {
@@ -1393,8 +1412,9 @@ class AgentThread {
   }
 }
 class ChatStreamProtocol {
-  constructor(streamConsumer) {
+  constructor(streamConsumer, onWrite) {
     this.streamConsumer = streamConsumer;
+    this.onWrite = onWrite;
   }
   _statistics = {
     sent: 0,
@@ -1405,43 +1425,45 @@ class ChatStreamProtocol {
       this._statistics.sent = 0;
     }
   };
-  static fromContext(ctx) {
-    return new ChatStreamProtocol(ctx.res);
+  static fromContext(ctx, onWrite) {
+    return new ChatStreamProtocol(ctx.res, onWrite);
   }
   with(conversation) {
-    const write = ({ type, body }) => {
+    const write = async ({ type, body }) => {
+      var _a;
       const { sessionId, from, username } = conversation;
       const data = `data: ${JSON.stringify({ sessionId, from, username, type, body })}
 
 `;
+      await ((_a = this.onWrite) == null ? void 0 : _a.call(this, data));
       this.streamConsumer.write(data);
       this._statistics.addSent(data.length);
     };
     return {
-      startStream: () => {
+      startStream: async () => {
         this._statistics.reset();
-        write({ type: "stream_start" });
+        await write({ type: "stream_start" });
       },
-      endStream: () => {
-        write({ type: "stream_end" });
+      endStream: async () => {
+        await write({ type: "stream_end" });
       },
-      subAgentCompleted: () => {
-        write({ type: "sub_agent_completed" });
+      subAgentCompleted: async () => {
+        await write({ type: "sub_agent_completed" });
       },
-      newMessage: (content) => {
-        write({ type: "new_message", body: content });
+      newMessage: async (content) => {
+        await write({ type: "new_message", body: content });
       },
-      content: (content) => {
-        write({ type: "content", body: content });
+      content: async (content) => {
+        await write({ type: "content", body: content });
       },
-      webSearch: (content) => {
-        write({ type: "web_search", body: content });
+      webSearch: async (content) => {
+        await write({ type: "web_search", body: content });
       },
-      reasoning: (content) => {
-        write({ type: "reasoning", body: content });
+      reasoning: async (content) => {
+        await write({ type: "reasoning", body: content });
       },
-      stopReasoning: () => {
-        write({
+      stopReasoning: async () => {
+        await write({
           type: "reasoning",
           body: {
             status: "stop",
@@ -1449,13 +1471,13 @@ class ChatStreamProtocol {
           }
         });
       },
-      toolCallChunks: (content) => {
-        write({ type: "tool_call_chunks", body: content });
+      toolCallChunks: async (content) => {
+        await write({ type: "tool_call_chunks", body: content });
       },
-      toolCalls: (content) => {
-        write({ type: "tool_calls", body: content });
+      toolCalls: async (content) => {
+        await write({ type: "tool_calls", body: content });
       },
-      toolCallStatus: ({
+      toolCallStatus: async ({
         toolCall,
         invokeStatus,
         status,
@@ -1464,7 +1486,7 @@ class ChatStreamProtocol {
         content,
         interruptAction
       }) => {
-        write({
+        await write({
           type: "tool_call_status",
           body: {
             toolCall,

package/dist/server/ai-employees/middleware/conversation.js

@@ -109,6 +109,9 @@ const conversationMiddleware = (aiEmployee, options) => {
         }
       });
     },
+    afterAgent: async () => {
+      aiEmployee.removeAbortController();
+    },
     beforeModel: async (state, runtime) => {
       var _a;
       const { messageId: messageId2 } = state;
@@ -159,7 +162,6 @@ const conversationMiddleware = (aiEmployee, options) => {
         if ((lastMessage == null ? void 0 : lastMessage.type) !== "ai") {
           return newState;
         }
-        aiEmployee.removeAbortController();
         if ((_a = runtime.signal) == null ? void 0 : _a.aborted) {
           return newState;
         }
@@ -180,19 +182,19 @@ const conversationMiddleware = (aiEmployee, options) => {
               fillToolCall(result, toolsMap, initializedToolCalls, toolCalls);
             }
           });
-          (_b = runtime.writer) == null ? void 0 : _b.call(runtime, {
+          if (toolCalls == null ? void 0 : toolCalls.length) {
+            (_b = runtime.writer) == null ? void 0 : _b.call(runtime, {
+              action: "initToolCalls",
+              body: { toolCalls },
+              currentConversation
+            });
+          }
+          (_c = runtime.writer) == null ? void 0 : _c.call(runtime, {
             action: "AfterAIMessageSaved",
             body: { id: aiMessage.id, messageId: newState.messageId },
             currentConversation
           });
         }
-        if (toolCalls == null ? void 0 : toolCalls.length) {
-          (_c = runtime.writer) == null ? void 0 : _c.call(runtime, {
-            action: "initToolCalls",
-            body: { toolCalls },
-            currentConversation
-          });
-        }
         return newState;
       } catch (e) {
         (_f = (_e = (_d = runtime.context) == null ? void 0 : _d.ctx) == null ? void 0 : _e.logger) == null ? void 0 : _f.error(e);

package/dist/server/collections/ai-conversations.js

@@ -97,6 +97,12 @@ var ai_conversations_default = (0, import_database.defineCollection)({
       type: "string",
       // chat, task
       defaultValue: "chat"
+    },
+    {
+      name: "read",
+      type: "boolean",
+      allowNull: false,
+      defaultValue: true
     }
   ]
 });

package/dist/server/manager/llm-stream-manager.d.ts

@@ -0,0 +1,37 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import PluginAIServer from '../plugin';
+export declare class LLMStreamCachedManager {
+    private readonly plugin;
+    private cachePromise?;
+    constructor(plugin: PluginAIServer);
+    getCached(sessionId: string): LLMStreamCached;
+    clear(sessionId: string): Promise<void>;
+    append(sessionId: string, chunk: string): Promise<void>;
+    stream(sessionId: string, options?: {
+        pollInterval?: number;
+        initialWaitTimeout?: number;
+    }): AsyncGenerator<string, void, void>;
+    private getChunks;
+    private getCache;
+    private withLock;
+    private getLockKey;
+}
+export declare class LLMStreamCached {
+    private readonly sessionId;
+    private readonly manager;
+    constructor(sessionId: string, manager: LLMStreamCachedManager);
+    clear(): Promise<void>;
+    append(chunk: string): Promise<void>;
+    skipped(): Promise<void>;
+    stream(options?: {
+        pollInterval?: number;
+        initialWaitTimeout?: number;
+    }): AsyncGenerator<string, void, void>;
+}

package/dist/server/manager/llm-stream-manager.js

@@ -0,0 +1,142 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var llm_stream_manager_exports = {};
+__export(llm_stream_manager_exports, {
+  LLMStreamCached: () => LLMStreamCached,
+  LLMStreamCachedManager: () => LLMStreamCachedManager
+});
+module.exports = __toCommonJS(llm_stream_manager_exports);
+var import_utils = require("@nocobase/utils");
+const CACHE_NAME = "ai-llm-stream-cache";
+const LOCK_KEY_PREFIX = "ai-llm-stream-lock";
+const WRITE_LOCK_TTL = 3e3;
+const CACHE_TTL = 10 * 60 * 1e3;
+const STREAM_END_MARK = '"type":"stream_end"';
+const SKIPPED_MARK = "__skipped__";
+const DEFAULT_POLL_INTERVAL = 50;
+const DEFAULT_INITIAL_WAIT_TIMEOUT = 1e3;
+class LLMStreamCachedManager {
+  constructor(plugin) {
+    this.plugin = plugin;
+  }
+  cachePromise;
+  getCached(sessionId) {
+    return new LLMStreamCached(sessionId, this);
+  }
+  async clear(sessionId) {
+    await this.withLock(sessionId, async () => {
+      const cache = await this.getCache();
+      await cache.del(sessionId);
+    });
+  }
+  async append(sessionId, chunk) {
+    await this.withLock(sessionId, async () => {
+      const cache = await this.getCache();
+      const chunks = await cache.get(sessionId) ?? [];
+      chunks.push(chunk);
+      await cache.set(sessionId, chunks, CACHE_TTL);
+    });
+  }
+  async *stream(sessionId, options) {
+    const pollInterval = (options == null ? void 0 : options.pollInterval) ?? DEFAULT_POLL_INTERVAL;
+    const initialWaitTimeout = (options == null ? void 0 : options.initialWaitTimeout) ?? DEFAULT_INITIAL_WAIT_TIMEOUT;
+    let offset = 0;
+    let waited = 0;
+    let completed = false;
+    while (!completed) {
+      const chunks = await this.getChunks(sessionId);
+      const lastSkippedIndex = chunks.lastIndexOf(SKIPPED_MARK);
+      if (lastSkippedIndex >= offset) {
+        offset = lastSkippedIndex + 1;
+      }
+      while (offset < chunks.length) {
+        const chunk = chunks[offset++];
+        yield chunk;
+        waited = 0;
+        if (chunk.includes(STREAM_END_MARK)) {
+          completed = true;
+          break;
+        }
+      }
+      if (completed) {
+        return;
+      }
+      if (!chunks.length) {
+        if (offset > 0) {
+          return;
+        }
+        if (waited >= initialWaitTimeout) {
+          return;
+        }
+        waited += pollInterval;
+      }
+      await (0, import_utils.sleep)(pollInterval);
+    }
+  }
+  async getChunks(sessionId) {
+    const cache = await this.getCache();
+    return await cache.get(sessionId) ?? [];
+  }
+  async getCache() {
+    this.cachePromise ??= this.plugin.app.cacheManager.createCache({
+      name: CACHE_NAME,
+      store: this.plugin.app.cacheManager.defaultStore
+    });
+    return this.cachePromise;
+  }
+  async withLock(sessionId, fn) {
+    return await this.plugin.app.lockManager.runExclusive(this.getLockKey(sessionId), fn, WRITE_LOCK_TTL);
+  }
+  getLockKey(sessionId) {
+    return `${LOCK_KEY_PREFIX}:${sessionId}`;
+  }
+}
+class LLMStreamCached {
+  constructor(sessionId, manager) {
+    this.sessionId = sessionId;
+    this.manager = manager;
+  }
+  async clear() {
+    await this.manager.clear(this.sessionId);
+  }
+  async append(chunk) {
+    await this.manager.append(this.sessionId, chunk);
+  }
+  async skipped() {
+    await this.append(SKIPPED_MARK);
+  }
+  async *stream(options) {
+    for await (const chunk of this.manager.stream(this.sessionId, options)) {
+      yield chunk;
+    }
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  LLMStreamCached,
+  LLMStreamCachedManager
+});

package/dist/server/plugin.d.ts

@@ -20,11 +20,13 @@ import { DocumentLoaders } from './document-loader';
 import type PluginFileManagerServer from '@nocobase/plugin-file-manager';
 import { SubAgentsDispatcher } from './ai-employees/sub-agents';
 import { KnowledgeBaseManager } from './ai-employees/ai-knowledge-base';
+import { LLMStreamCachedManager } from './manager/llm-stream-manager';
 export declare class PluginAIServer extends Plugin {
     features: AIPluginFeatureManagerImpl;
     aiManager: AIManager;
     aiEmployeesManager: AIEmployeesManager;
     aiConversationsManager: AIConversationsManager;
+    llmStreamCachedManager: LLMStreamCachedManager;
     builtInManager: BuiltInManager;
     aiContextDatasourceManager: AIContextDatasourceManager;
     aiCodingManager: AICodingManager;

package/dist/server/plugin.js

@@ -77,11 +77,13 @@ var import_mimo = require("./llm-providers/mimo");
 var import_sub_agents = require("./ai-employees/sub-agents");
 var import_employee = require("./workflow/nodes/employee");
 var import_ai_knowledge_base = require("./ai-employees/ai-knowledge-base");
+var import_llm_stream_manager = require("./manager/llm-stream-manager");
 class PluginAIServer extends import_server.Plugin {
   features = new import_ai_feature_manager.AIPluginFeatureManagerImpl();
   aiManager = new import_ai_manager.AIManager(this);
   aiEmployeesManager = new import_ai_employees_manager.AIEmployeesManager(this);
   aiConversationsManager = new import_ai_conversations.AIConversationsManager(this);
+  llmStreamCachedManager = new import_llm_stream_manager.LLMStreamCachedManager(this);
   builtInManager = new import_built_in_manager.BuiltInManager(this);
   aiContextDatasourceManager = new import_ai_context_datasource_manager.AIContextDatasourceManager(this);
   aiCodingManager = new import_ai_coding_manager.AICodingManager(this);
@@ -135,6 +137,7 @@ class PluginAIServer extends import_server.Plugin {
     this.registerWorkflow();
     this.registerWorkContextResolveStrategy();
     (0, import_employee.registerAIEmployeeTaskNotification)(this);
+    (0, import_ai_conversations.registerAIConversationReadNotification)(this);
     (0, import_employee.registerOnJobAbortedHandler)(this);
   }
   registerLLMProviders() {

package/dist/server/resource/aiConversations.d.ts

@@ -7,10 +7,17 @@
  * For more information, please refer to: https://www.nocobase.com/agreement.
  */
 import { Context, Next } from '@nocobase/actions';
+declare function parallelConversationsLimit(ctx: Context, next: Next): Promise<never>;
 declare const _default: {
     name: string;
+    middlewares: {
+        only: string[];
+        handler: typeof parallelConversationsLimit;
+    }[];
     actions: {
         list(ctx: Context, next: Next): Promise<void>;
+        unreadCount(ctx: Context, next: Next): Promise<never>;
+        unreadCounts(ctx: Context, next: Next): Promise<never>;
         create(ctx: Context, next: Next): Promise<never>;
         update(ctx: Context, next: Next): Promise<never>;
         updateOptions(ctx: Context, next: Next): Promise<never>;
@@ -19,6 +26,7 @@ declare const _default: {
         updateToolArgs(ctx: Context, next: Next): Promise<any>;
         sendMessages(ctx: Context, next: Next): Promise<any>;
         abort(ctx: Context, next: Next): Promise<never>;
+        resumeStream(ctx: Context, next: Next): Promise<never>;
         resendMessages(ctx: Context, next: Next): Promise<any>;
         updateUserDecision(ctx: Context, next: Next): Promise<never>;
         resumeToolCall(ctx: Context, next: Next): Promise<any>;