@cloudbase/agent-adapter-adp 0.0.16 → 0.0.19

package/README.md

@@ -15,6 +15,7 @@ npm install @cloudbase/agent-agents @cloudbase/agent-adapter-adp
 - **Thinking Events**: Support for thinking/reasoning process events from the model
 - **Workflow Integration**: Support for ADP workflows with tool call events
 - **Custom Variables**: Pass custom parameters to workflows and knowledge base
+- **File Upload**: Upload images and documents to COS for multimodal conversations (**requires `enableUpload: true`**)
 
 ## Environment Variables
 
@@ -121,6 +122,55 @@ observable.subscribe({
 });
 ```
 
+### With File Upload (Images and Documents)
+
+When `enableUpload` is **set to `true`**, you can send images and documents in messages. Files are automatically uploaded to Tencent Cloud COS and processed by ADP.
+
+**Note:** File upload requires Tencent Cloud credentials (`TENCENTCLOUD_SECRETID` and `TENCENTCLOUD_SECRETKEY`).
+
+```typescript
+import { AdpAgent } from "@cloudbase/agent-adapter-adp";
+import { randomUUID } from "crypto";
+import * as fs from "fs";
+
+const agent = new AdpAgent({
+  name: "my-adp-agent",
+  adpConfig: {
+    appKey: "your-app-key",
+    enableUpload: true, // Enable file upload
+    credential: {
+      secretId: "your-secret-id",
+      secretKey: "your-secret-key",
+    },
+  },
+});
+
+const observable = agent.run({
+  runId: randomUUID(),
+  threadId: randomUUID(),
+  messages: [
+    {
+      id: randomUUID(),
+      role: "user",
+      content: [
+        { type: "text", text: "What's in this image?" },
+        {
+          type: "binary",
+          mimeType: "image/png",
+          filename: "screenshot.png",
+          data: fs.readFileSync("./screenshot.png"),
+        },
+      ],
+    },
+  ],
+});
+```
+
+**Supported file types:**
+
+- **Images**: `image/png`, `image/jpeg`, `image/bmp`
+- **Documents**: `application/pdf`, `application/msword`, `application/vnd.openxmlformats-officedocument.wordprocessingml.document`, `application/vnd.ms-powerpoint`, `application/vnd.openxmlformats-officedocument.presentationml.presentation`, `text/plain`
+
 ## API Reference
 
 ### AdpAgent
@@ -146,9 +196,11 @@ interface AdpConfig {
     token?: string; // Session token (optional if TENCENTCLOUD_SESSIONTOKEN env is set)
   };
   historyCount?: number; // Number of history messages to retrieve (reserved)
+  enableUpload?: boolean; // Enable file upload for images and documents (default: false)
   request?: {
     baseUrl?: string; // Base URL for ADP API (default: https://wss.lke.cloud.tencent.com)
     endpoint?: string; // API endpoint (default: /v1/qbot/chat/sse)
+    docParseEndpoint?: string; // Document parse API endpoint (default: /v1/qbot/chat/docParse)
     body?: Partial<AdpChatRequest>; // Additional request body options
   };
 }
@@ -187,6 +239,7 @@ The adapter emits the following AG-UI events:
 - `axios`: HTTP client for API requests
 - `rxjs`: Reactive extensions for JavaScript
 - `tencentcloud-sdk-nodejs-lke`: Tencent Cloud LKE SDK
+- `cos-nodejs-sdk-v5`: Tencent Cloud COS SDK (for file upload)
 
 ## Related Resources
 

package/dist/index.d.mts

@@ -59,8 +59,10 @@ type AdpConfig = {
         baseUrl?: string;
         endpoint?: string;
         body?: Partial<AdpChatRequest>;
+        docParseEndpoint?: string;
     };
     historyCount?: number;
+    enableUpload?: boolean;
     appKey?: string;
     credential?: {
         secretId?: string;
@@ -282,6 +284,7 @@ declare class AdpAgent extends AbstractAgent {
     private reqLkeClient;
     protected adpConfig: AdpConfig;
     private finalAppKey;
+    private finalCloudCredential;
     constructor(config: AgentConfig & {
         adpConfig: AdpConfig;
     });

package/dist/index.d.ts

@@ -59,8 +59,10 @@ type AdpConfig = {
         baseUrl?: string;
         endpoint?: string;
         body?: Partial<AdpChatRequest>;
+        docParseEndpoint?: string;
     };
     historyCount?: number;
+    enableUpload?: boolean;
     appKey?: string;
     credential?: {
         secretId?: string;
@@ -282,6 +284,7 @@ declare class AdpAgent extends AbstractAgent {
     private reqLkeClient;
     protected adpConfig: AdpConfig;
     private finalAppKey;
+    private finalCloudCredential;
     constructor(config: AgentConfig & {
         adpConfig: AdpConfig;
     });

package/dist/index.js

@@ -129,18 +129,20 @@ var AdpAgent = class extends import_client.AbstractAgent {
   constructor(config) {
     super(config);
     this.finalAppKey = "";
+    this.finalCloudCredential = {};
     this.adpConfig = config.adpConfig;
     this.finalAppKey = this.adpConfig.appKey || this.adpConfig.request?.body?.botAppKey || process.env.ADP_APP_KEY || "";
     this.reqAppClient = import_axios.default.create({
       baseURL: this.adpConfig.request?.baseUrl || "https://wss.lke.cloud.tencent.com"
     });
     const LkeClient = import_tencentcloud_sdk_nodejs_lke.lke.v20231130.Client;
+    this.finalCloudCredential = {
+      secretId: this.adpConfig.credential?.secretId || process.env.TENCENTCLOUD_SECRETID,
+      secretKey: this.adpConfig.credential?.secretKey || process.env.TENCENTCLOUD_SECRETKEY,
+      token: this.adpConfig.credential?.token || process.env.TENCENTCLOUD_SESSIONTOKEN
+    };
     this.reqLkeClient = new LkeClient({
-      credential: {
-        secretId: this.adpConfig.credential?.secretId || process.env.TENCENTCLOUD_SECRETID,
-        secretKey: this.adpConfig.credential?.secretKey || process.env.TENCENTCLOUD_SECRETKEY,
-        token: this.adpConfig.credential?.token || process.env.TENCENTCLOUD_SESSIONTOKEN
-      }
+      credential: this.finalCloudCredential
     });
   }
   generateRequestBody({
@@ -169,9 +171,11 @@ var AdpAgent = class extends import_client.AbstractAgent {
     });
   }
   async _run(subscriber, input) {
+    let thinkingMessageSet = /* @__PURE__ */ new Set();
+    let thinkFinishedMessageSet = /* @__PURE__ */ new Set();
     try {
-      const { runId, threadId: _threadId } = input;
-      const threadId = _threadId || (0, import_crypto.randomUUID)();
+      const { runId } = input;
+      const threadId = input.threadId || (0, import_crypto.randomUUID)();
       subscriber.next({
         type: import_client.EventType.RUN_STARTED,
         runId,
@@ -183,19 +187,26 @@ var AdpAgent = class extends import_client.AbstractAgent {
           "MISSING_APP_KEY"
         );
       }
-      const { messages: docExtractedMessages, fileInfos } = await this.extractDocuments(input, subscriber);
-      const { message, trimmed } = await this.convertAGUIMessagesToAdpMessages(docExtractedMessages);
+      const latestUserMessage = input.messages.filter((m) => m.role === "user").pop();
+      if (!latestUserMessage) {
+        throw new AdpAgentError(
+          "Message content format error, or empty content",
+          "INVALID_MESSAGE_FORMAT"
+        );
+      }
+      const { message: docExtractedMessage, fileInfos } = await this.extractDocuments(latestUserMessage, threadId, subscriber);
+      const message = await this.convertAGUIMessagesToAdpMessages(docExtractedMessage);
       if (!message) {
         throw new AdpAgentError(
           "Message content format error, or empty content",
           "INVALID_MESSAGE_FORMAT"
         );
       }
-      if (trimmed > 0) {
+      if (input.messages.length > 1) {
         subscriber.next({
           type: import_client.EventType.RAW,
           rawEvent: {
-            message: `ADP handles message history itself, so that a total of ${trimmed} messages before and including last assistant message will be trimmed.`,
+            message: `ADP handles message history itself, so that a total of ${input.messages.length - 1} messages before and including last assistant message will be trimmed.`,
             type: "warn"
           }
         });
@@ -214,7 +225,6 @@ var AdpAgent = class extends import_client.AbstractAgent {
       let buffer = "";
       let interruptRequested = false;
       let thinkingStart = false;
-      let thinkingMessageSet = /* @__PURE__ */ new Set();
       for await (const chunk of sseStream) {
         buffer += chunk.toString();
         const parts = buffer.split("\n\n");
@@ -241,6 +251,22 @@ var AdpAgent = class extends import_client.AbstractAgent {
             }
             switch (data.type) {
               case "reply": {
+                const messageId = data.payload.record_id;
+                const isFinal = data.payload.is_final;
+                if (thinkingStart) {
+                  thinkingStart = false;
+                  for (const index of thinkingMessageSet) {
+                    subscriber.next({
+                      type: import_client.EventType.THINKING_TEXT_MESSAGE_END,
+                      messageId: `${messageId}-think-${index}`
+                    });
+                  }
+                  thinkingMessageSet.clear();
+                  subscriber.next({
+                    type: import_client.EventType.THINKING_END,
+                    messageId
+                  });
+                }
                 if (data.payload.is_from_self) {
                   if (data.payload.is_evil) {
                     throw new AdpAgentError(
@@ -251,8 +277,6 @@ var AdpAgent = class extends import_client.AbstractAgent {
                     continue;
                   }
                 }
-                const messageId = data.payload.record_id;
-                const isFinal = data.payload.is_final;
                 data.payload.content = data.payload.content.replace(
                   /\\n/g,
                   "\n\n"
@@ -332,51 +356,47 @@ var AdpAgent = class extends import_client.AbstractAgent {
                   });
                 }
                 data.payload.procedures.forEach((procedure) => {
-                  if (!thinkingMessageSet.has(messageId)) {
-                    thinkingMessageSet.add(messageId);
+                  const index = procedure.index.toString();
+                  if (!thinkingMessageSet.has(index) && !thinkFinishedMessageSet.has(index)) {
+                    thinkingMessageSet.add(index);
                     subscriber.next({
                       type: import_client.EventType.THINKING_TEXT_MESSAGE_START,
-                      messageId,
+                      messageId: `${messageId}-think-${index}`,
                       delta: procedure.debugging.content
                     });
                   } else {
                     if (procedure.status === "processing") {
                       subscriber.next({
                         type: import_client.EventType.THINKING_TEXT_MESSAGE_CONTENT,
-                        messageId,
+                        messageId: `${messageId}-think-${index}`,
                         delta: procedure.debugging.content
                       });
                     } else {
-                      thinkingMessageSet.delete(messageId);
-                      subscriber.next({
-                        type: import_client.EventType.THINKING_TEXT_MESSAGE_END,
-                        messageId
-                      });
+                      thinkingMessageSet.delete(index);
+                      if (!thinkFinishedMessageSet.has(index)) {
+                        thinkFinishedMessageSet.add(index);
+                        subscriber.next({
+                          type: import_client.EventType.THINKING_TEXT_MESSAGE_END,
+                          messageId: `${messageId}-think-${index}`
+                        });
+                      }
                     }
                   }
                 });
-                const allFinished = data.payload.procedures.every(
-                  (procedure) => procedure.status !== "processing"
-                );
-                if (allFinished) {
-                  thinkingStart = false;
-                  thinkingMessageSet.clear();
-                  subscriber.next({
-                    type: import_client.EventType.THINKING_END,
-                    messageId
-                  });
-                }
                 break;
               }
               case "error": {
-                console.error(JSON.stringify(data));
+                console.error(
+                  "[ERROR] ADP throws error: ",
+                  JSON.stringify(data)
+                );
                 throw new AdpAgentError(
                   data.error.message,
                   data.error.code ? `ADP_ERROR_${data.error.code}` : "ADP_ERROR_-1"
                 );
               }
               case "token_stat": {
-                console.debug(JSON.stringify(data));
+                console.debug("[DEBUG] ADP token stat: ", JSON.stringify(data));
                 break;
               }
               case "reference": {
@@ -406,7 +426,7 @@ var AdpAgent = class extends import_client.AbstractAgent {
       }
       subscriber.complete();
     } catch (e) {
-      console.error(JSON.stringify(e));
+      console.error("[ERROR] Uncaught error: ", JSON.stringify(e));
       let code = "UNKNOWN_ERROR";
       let message = JSON.stringify(e);
       if (e instanceof import_axios.AxiosError) {
@@ -422,21 +442,33 @@ var AdpAgent = class extends import_client.AbstractAgent {
         message: `Sorry, an error occurred while running the agent: Error code ${code}, ${message}`
       });
       subscriber.complete();
+    } finally {
+      thinkingMessageSet.clear();
+      thinkFinishedMessageSet.clear();
     }
   }
-  async convertAGUIMessagesToAdpMessages(messages) {
+  async convertAGUIMessagesToAdpMessages(message) {
     let result = "";
-    let trimmed = messages.length;
-    for (const message of messages.reverse()) {
-      if (message.role === "assistant") {
-        break;
-      }
-      if (message.role === "user") {
-        trimmed--;
-        let content = "";
-        if (typeof message.content === "string") {
-          content = message.content;
-        } else {
+    if (message.role === "user") {
+      let content = "";
+      if (typeof message.content === "string") {
+        content = message.content;
+      } else {
+        if (this.adpConfig.enableUpload) {
+          if (!this.finalCloudCredential.token) {
+            if (!this.finalCloudCredential.secretId) {
+              throw new AdpAgentError(
+                "TENCENTCLOUD_SECRETID is required, check your env variables or config passed with the adapter",
+                "MISSING_SECRET_ID"
+              );
+            }
+            if (!this.finalCloudCredential.secretKey) {
+              throw new AdpAgentError(
+                "TENCENTCLOUD_SECRETKEY is required, check your env variables or config passed with the adapter",
+                "MISSING_SECRET_KEY"
+              );
+            }
+          }
           const imageMap = /* @__PURE__ */ new Map();
           const imagesToUpload = [];
           message.content.forEach((item) => {
@@ -479,143 +511,158 @@ var AdpAgent = class extends import_client.AbstractAgent {
             }
           }, "").trim();
         }
-        result = `${message.role}: ${content}
+      }
+      result = `${message.role}: ${content}
 ${result}`;
-      } else {
-        result = `${message.role}: ${message.content}
+    } else {
+      result = `${message.role}: ${message.content}
 ${result}`;
-      }
     }
-    return { message: result.trim(), trimmed };
+    return result.trim();
   }
-  async extractDocuments(input, subscriber) {
-    const { messages, runId, threadId } = input;
+  async extractDocuments(message, threadId, subscriber) {
     const documentFiles = [];
-    const newMessages = messages.map((msg) => {
-      if (msg.role === "user" && Array.isArray(msg.content)) {
-        let newContent = [];
-        msg.content.forEach((item) => {
-          if (item.type === "text") {
+    let newMessage;
+    if (message.role === "user" && Array.isArray(message.content)) {
+      let newContent = [];
+      message.content.forEach((item) => {
+        if (item.type === "text") {
+          newContent.push(item);
+        } else if (item.type === "binary") {
+          if (Object.keys(DOCUMENT_MIME_TYPES).includes(item.mimeType)) {
+            documentFiles.push(item);
+          } else {
             newContent.push(item);
-          } else if (item.type === "binary") {
-            if (Object.keys(DOCUMENT_MIME_TYPES).includes(item.mimeType)) {
-              documentFiles.push(item);
-            } else {
-              newContent.push(item);
-            }
           }
-        });
-        return {
-          ...msg,
-          content: newContent
-        };
-      } else return msg;
-    });
+        }
+      });
+      newMessage = {
+        ...message,
+        content: newContent
+      };
+    } else newMessage = message;
     const fileInfos = [];
     const successedFiles = [];
     const failedFiles = [];
-    if (documentFiles.length) {
-      try {
-        await this.uploadToCos(
-          documentFiles,
-          (data, cosParams, file) => {
-            successedFiles.push({ data, cosParams, file });
-          },
-          (fileName, error) => {
-            failedFiles.push({ fileName, error });
-          }
-        );
-      } catch (e) {
+    if (this.adpConfig.enableUpload) {
+      if (!this.finalCloudCredential.token) {
+        if (!this.finalCloudCredential.secretId) {
+          throw new AdpAgentError(
+            "TENCENTCLOUD_SECRETID is required, check your env variables or config passed with the adapter",
+            "MISSING_SECRET_ID"
+          );
+        }
+        if (!this.finalCloudCredential.secretKey) {
+          throw new AdpAgentError(
+            "TENCENTCLOUD_SECRETKEY is required, check your env variables or config passed with the adapter",
+            "MISSING_SECRET_KEY"
+          );
+        }
       }
-    }
-    if (successedFiles.length) {
-      successedFiles.forEach(async ({ data: cosData, cosParams, file }) => {
-        const extName = MIME_TYPES[file.mimeType];
-        const requestBody = {
-          sessionId: threadId,
-          botAppKey: this.finalAppKey,
-          requestId: (0, import_crypto.randomUUID)(),
-          cosBucket: cosParams.Bucket,
-          fileType: extName,
-          fileName: file.filename,
-          cosUrl: cosParams.UploadPath,
-          cosHash: cosData.headers?.["x-cos-hash-crc64ecma"] || "",
-          eTag: cosData.ETag,
-          size: file.data?.length.toString() || "0"
-        };
-        const response = await this.reqAppClient.post(
-          this.adpConfig.request?.endpoint || "/v1/qbot/chat/docParse",
-          camelToSnakeKeys(requestBody),
-          { responseType: "stream" }
-        );
-        const sseStream = response.data;
-        let buffer = "";
-        for await (const chunk of sseStream) {
-          buffer += chunk.toString();
-          const parts = buffer.split("\n\n");
-          buffer = parts.pop() || "";
-          for (const part of parts) {
-            if (!part.trim()) continue;
-            const event = { data: "", event: "" };
-            for (const line of part.split("\n")) {
-              if (line.startsWith("data:")) {
-                event.data += line.slice(5);
-              } else if (line.startsWith("event:")) {
-                event.event = line.slice(6);
-              }
+      if (documentFiles.length) {
+        try {
+          await this.uploadToCos(
+            documentFiles,
+            (data, cosParams, file) => {
+              successedFiles.push({ data, cosParams, file });
+            },
+            (fileName, error) => {
+              failedFiles.push({ fileName, error });
             }
-            if (event.data) {
-              let data;
-              try {
-                data = JSON.parse(event.data);
-              } catch (e) {
-                throw new AdpAgentError(
-                  `ADP returned invalid data: ${event.data}`,
-                  "INVALID_DATA"
-                );
+          );
+        } catch (e) {
+          console.error("Document upload failed: ", JSON.stringify(e));
+        }
+      }
+      if (successedFiles.length) {
+        for (const { data: cosData, cosParams, file } of successedFiles) {
+          const extName = MIME_TYPES[file.mimeType];
+          const requestBody = {
+            sessionId: threadId,
+            botAppKey: this.finalAppKey,
+            requestId: (0, import_crypto.randomUUID)(),
+            cosBucket: cosParams.Bucket,
+            fileType: extName,
+            fileName: file.filename,
+            cosUrl: cosParams.UploadPath,
+            cosHash: cosData.headers?.["x-cos-hash-crc64ecma"] || "",
+            eTag: cosData.ETag,
+            size: file.data?.length.toString() || "0"
+          };
+          const response = await this.reqAppClient.post(
+            this.adpConfig.request?.docParseEndpoint || "/v1/qbot/chat/docParse",
+            camelToSnakeKeys(requestBody),
+            { responseType: "stream" }
+          );
+          const sseStream = response.data;
+          let buffer = "";
+          for await (const chunk of sseStream) {
+            buffer += chunk.toString();
+            const parts = buffer.split("\n\n");
+            buffer = parts.pop() || "";
+            for (const part of parts) {
+              if (!part.trim()) continue;
+              const event = { data: "", event: "" };
+              for (const line of part.split("\n")) {
+                if (line.startsWith("data:")) {
+                  event.data += line.slice(5);
+                } else if (line.startsWith("event:")) {
+                  event.event = line.slice(6);
+                }
               }
-              switch (data.type) {
-                case "parsing": {
-                  subscriber.next({
-                    type: import_client.EventType.RAW,
-                    rawEvent: {
-                      message: `Parsing document ${file.filename}: ${data.payload.process}%`,
-                      type: "info"
-                    }
-                  });
-                  if (data.payload.is_final) {
-                    if (data.payload.error_message) {
-                      subscriber.next({
-                        type: import_client.EventType.RAW,
-                        rawEvent: {
-                          message: `Parsing document ${file.filename} failed: ${data.payload.error_message}`,
-                          type: "error"
-                        }
-                      });
-                    } else {
-                      const fileNameNoExt = file.filename.split(".").splice(0, -1).join(".");
-                      fileInfos.push({
-                        docId: data.payload.doc_id,
-                        fileName: fileNameNoExt,
-                        fileType: extName,
-                        fileSize: file.data?.length.toString() || "0",
-                        fileUrl: `https://${cosData.Location}`
-                      });
+              if (event.data) {
+                let data;
+                try {
+                  data = JSON.parse(event.data);
+                } catch (e) {
+                  throw new AdpAgentError(
+                    `ADP returned invalid data: ${event.data}`,
+                    "INVALID_DATA"
+                  );
+                }
+                switch (data.type) {
+                  case "parsing": {
+                    subscriber.next({
+                      type: import_client.EventType.RAW,
+                      rawEvent: {
+                        message: `Parsing document ${file.filename}: ${data.payload.process}%`,
+                        type: "info"
+                      }
+                    });
+                    if (data.payload.is_final) {
+                      if (data.payload.error_message) {
+                        subscriber.next({
+                          type: import_client.EventType.RAW,
+                          rawEvent: {
+                            message: `Parsing document ${file.filename} failed: ${data.payload.error_message}`,
+                            type: "error"
+                          }
+                        });
+                      } else {
+                        const fileNameNoExt = file.filename.split(".").splice(0, -1).join(".");
+                        fileInfos.push({
+                          docId: data.payload.doc_id,
+                          fileName: fileNameNoExt,
+                          fileType: extName,
+                          fileSize: file.data?.length.toString() || "0",
+                          fileUrl: `https://${cosData.Location}`
+                        });
+                      }
                     }
+                    break;
+                  }
+                  default: {
+                    break;
                   }
-                  break;
-                }
-                default: {
-                  break;
                 }
               }
             }
           }
         }
-      });
+      }
     }
     return {
-      messages: newMessages,
+      message: newMessage,
       fileInfos,
       failedFiles
     };