@cognipeer/agent-sdk 0.8.2 → 0.8.4

package/dist/index.cjs

@@ -4573,6 +4573,7 @@ function resolveUserQuestionState(state, resolution) {
     ctx
   };
 }
+var ASK_USER_TOOL_NAME = "ask_user_question";
 var DEFAULT_DESCRIPTION_WITH_FREETEXT = "Pause the agent and surface a structured prompt to the human user when you genuinely cannot proceed without their input. Group up to 4 closely related questions into a single call. Prefer concrete options the user can pick over open-ended free-text \u2014 list `options` whenever you can anticipate the choices. The user may type a custom answer for any question. Do not use this tool for things you can deduce from the conversation, the system prompt, or another tool. After the user answers, their selections come back as the tool result.";
 var DEFAULT_DESCRIPTION_NO_FREETEXT = "Pause the agent and surface a structured multi-choice prompt to the human user when you genuinely cannot proceed without their input. Group up to 4 closely related questions into a single call. Every question MUST include at least two `options` \u2014 free-text answers are disabled for this agent. Do not use this tool for things you can deduce from the conversation, the system prompt, or another tool. After the user picks, their selections come back as the tool result.";
 function buildOptionSchema() {
@@ -4608,7 +4609,7 @@ function createAskUserQuestionTool(stateRef, config) {
   const description = config.promptOverride ?? (allowFreeText ? DEFAULT_DESCRIPTION_WITH_FREETEXT : DEFAULT_DESCRIPTION_NO_FREETEXT);
   const questionSchema = buildQuestionSchema(allowFreeText);
   const askUser = createTool({
-    name: "ask_user_question",
+    name: ASK_USER_TOOL_NAME,
     description,
     schema: zod.z.object({
       questions: zod.z.array(questionSchema).min(1).max(4).describe("1-4 related questions to ask in a single pause.")
@@ -4624,7 +4625,7 @@ function createAskUserQuestionTool(stateRef, config) {
       const entry = {
         id: nanoid.nanoid(),
         toolCallId,
-        toolName: "ask_user_question",
+        toolName: ASK_USER_TOOL_NAME,
         questions,
         status: "pending",
         requestedAt: (/* @__PURE__ */ new Date()).toISOString(),
@@ -4980,14 +4981,15 @@ function createAgent(opts) {
   })();
   const askUserOnQuestion = typeof opts.humanInTheLoop?.askUser === "object" ? opts.humanInTheLoop?.askUser?.onQuestion : void 0;
   const askUserStateRef = askUserConfig ? { pendingUserQuestions: void 0, ctx: void 0, __onEvent: void 0, __currentToolCallId: void 0 } : void 0;
-  if (askUserConfig && askUserStateRef) {
+  const hasTool = (name) => toolsBase.some((tool) => tool?.name === name);
+  if (askUserConfig && askUserStateRef && !hasTool(ASK_USER_TOOL_NAME)) {
     toolsBase.push(createAskUserQuestionTool(askUserStateRef, askUserConfig));
   }
   const soManager = opts.outputSchema ? new StructuredOutputManager(opts.outputSchema, resolveStrategy(opts.model)) : void 0;
   if (soManager) {
     const modelCapabilities = getModelCapabilities(opts.model);
     const responseTool = soManager.getResponseTool();
-    if (responseTool && modelCapabilities.structuredOutput !== "native") {
+    if (responseTool && modelCapabilities.structuredOutput !== "native" && !hasTool(responseTool.name)) {
       toolsBase.push(responseTool);
     }
   }
@@ -8600,6 +8602,78 @@ function resolveBudget(reasoning, effortBudget) {
   return void 0;
 }
 
+// src/providers/utils/media.ts
+var EXTENSION_MIME = {
+  jpg: "image/jpeg",
+  jpeg: "image/jpeg",
+  png: "image/png",
+  gif: "image/gif",
+  webp: "image/webp",
+  svg: "image/svg+xml",
+  bmp: "image/bmp",
+  tif: "image/tiff",
+  tiff: "image/tiff",
+  heic: "image/heic",
+  heif: "image/heif",
+  pdf: "application/pdf",
+  csv: "text/csv",
+  txt: "text/plain",
+  md: "text/markdown",
+  html: "text/html",
+  doc: "application/msword",
+  docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+  xls: "application/vnd.ms-excel",
+  xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+  ppt: "application/vnd.ms-powerpoint",
+  pptx: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+  mp3: "audio/mpeg",
+  mpga: "audio/mpeg",
+  wav: "audio/wav",
+  ogg: "audio/ogg",
+  flac: "audio/flac",
+  aac: "audio/aac",
+  m4a: "audio/mp4",
+  webm: "audio/webm",
+  mp4: "video/mp4",
+  mov: "video/quicktime",
+  avi: "video/x-msvideo"
+};
+function inferMediaTypeFromUrl(urlOrName) {
+  if (!urlOrName) return void 0;
+  const cleaned = urlOrName.split("?")[0].split("#")[0];
+  const ext = cleaned.split(".").pop()?.toLowerCase();
+  if (!ext || ext === cleaned.toLowerCase()) return void 0;
+  return EXTENSION_MIME[ext];
+}
+function audioMimeToOpenAIFormat(mediaType) {
+  const m = (mediaType ?? "").toLowerCase();
+  if (m.includes("wav")) return "wav";
+  return "mp3";
+}
+var BEDROCK_DOC_FORMATS = {
+  "application/pdf": "pdf",
+  "text/csv": "csv",
+  "application/msword": "doc",
+  "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
+  "application/vnd.ms-excel": "xls",
+  "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",
+  "text/html": "html",
+  "text/plain": "txt",
+  "text/markdown": "md"
+};
+function documentMimeToBedrockFormat(mediaType, fileName) {
+  if (mediaType && BEDROCK_DOC_FORMATS[mediaType.toLowerCase()]) {
+    return BEDROCK_DOC_FORMATS[mediaType.toLowerCase()];
+  }
+  const inferred = inferMediaTypeFromUrl(fileName);
+  if (inferred && BEDROCK_DOC_FORMATS[inferred]) return BEDROCK_DOC_FORMATS[inferred];
+  return void 0;
+}
+function sanitizeBedrockDocName(name, fallback = "document") {
+  const cleaned = (name ?? "").replace(/\.[^.]+$/, "").replace(/[^a-zA-Z0-9\s\-()[\]]/g, " ").replace(/\s+/g, " ").trim();
+  return cleaned || fallback;
+}
+
 // src/providers/openai.ts
 var OpenAIProvider = class extends BaseProvider {
   constructor(config) {
@@ -8822,6 +8896,27 @@ var OpenAIProvider = class extends BaseProvider {
         const url = src.type === "url" ? src.url : `data:${src.mediaType};base64,${src.data}`;
         return { type: "input_image", image_url: url };
       }
+      if (part.type === "file") {
+        const src = part.source;
+        if (src.type === "base64") {
+          return {
+            type: "input_file",
+            filename: part.fileName ?? "file",
+            file_data: `data:${src.mediaType};base64,${src.data}`
+          };
+        }
+        return { type: "input_file", file_url: src.url };
+      }
+      if (part.type === "audio") {
+        const src = part.source;
+        if (src.type === "base64") {
+          return {
+            type: "input_audio",
+            input_audio: { data: src.data, format: audioMimeToOpenAIFormat(src.mediaType) }
+          };
+        }
+        return { type: textType, text: `[audio: ${src.url}]` };
+      }
       return { type: textType, text: "" };
     });
     return { role, content };
@@ -8906,6 +9001,29 @@ var OpenAIProvider = class extends BaseProvider {
           const url = src.type === "url" ? src.url : `data:${src.mediaType};base64,${src.data}`;
           return { type: "image_url", image_url: { url } };
         }
+        if (part.type === "file") {
+          const src = part.source;
+          if (src.type === "base64") {
+            return {
+              type: "file",
+              file: {
+                filename: part.fileName ?? "file",
+                file_data: `data:${src.mediaType};base64,${src.data}`
+              }
+            };
+          }
+          return { type: "text", text: `[file: ${src.url}]` };
+        }
+        if (part.type === "audio") {
+          const src = part.source;
+          if (src.type === "base64") {
+            return {
+              type: "input_audio",
+              input_audio: { data: src.data, format: audioMimeToOpenAIFormat(src.mediaType) }
+            };
+          }
+          return { type: "text", text: `[audio: ${src.url}]` };
+        }
         return part;
       });
     }
@@ -9303,6 +9421,22 @@ var AnthropicProvider = class extends BaseProvider {
             source: { type: "url", url: src.url }
           };
         }
+        if (part.type === "file") {
+          const src = part.source;
+          const title = part.fileName ? { title: part.fileName } : {};
+          if (src.type === "base64") {
+            return {
+              type: "document",
+              source: { type: "base64", media_type: src.mediaType, data: src.data },
+              ...title
+            };
+          }
+          return { type: "document", source: { type: "url", url: src.url }, ...title };
+        }
+        if (part.type === "audio") {
+          const src = part.source;
+          return { type: "text", text: src.type === "url" ? `[audio: ${src.url}]` : "[audio attachment omitted: not supported by this model]" };
+        }
         return part;
       });
     }
@@ -9770,6 +9904,24 @@ var BedrockProvider = class extends BaseProvider {
               }
             });
           }
+        } else if (part.type === "file") {
+          const src = part.source;
+          const format = documentMimeToBedrockFormat(src.mediaType, part.fileName);
+          if (src.type === "base64" && format) {
+            content.push({
+              document: {
+                format,
+                name: sanitizeBedrockDocName(part.fileName),
+                source: { bytes: src.data }
+              }
+            });
+          } else {
+            const ref = src.type === "url" ? src.url : part.fileName ?? "attachment";
+            content.push({ text: `[file: ${ref}]` });
+          }
+        } else if (part.type === "audio") {
+          const src = part.source;
+          content.push({ text: src.type === "url" ? `[audio: ${src.url}]` : "[audio attachment omitted: not supported by this model]" });
         }
       }
     }
@@ -10113,7 +10265,25 @@ var VertexProvider = class extends BaseProvider {
             });
           } else {
             parts.push({
-              fileData: { mimeType: "image/jpeg", fileUri: src.url }
+              fileData: {
+                mimeType: src.mediaType ?? inferMediaTypeFromUrl(src.url) ?? "image/jpeg",
+                fileUri: src.url
+              }
+            });
+          }
+        } else if (part.type === "file" || part.type === "audio") {
+          const src = part.source;
+          const fallbackMime = part.type === "audio" ? "audio/mpeg" : "application/pdf";
+          if (src.type === "base64") {
+            parts.push({
+              inlineData: { mimeType: src.mediaType ?? fallbackMime, data: src.data }
+            });
+          } else {
+            parts.push({
+              fileData: {
+                mimeType: src.mediaType ?? inferMediaTypeFromUrl(src.url) ?? inferMediaTypeFromUrl(part.fileName) ?? fallbackMime,
+                fileUri: src.url
+              }
             });
           }
         }
@@ -10458,9 +10628,50 @@ function convertContent(content) {
         }
       }
     }
+    if (part.type === "file" || part.type === "document") {
+      const source = extractBinarySource(part);
+      if (source) {
+        const fileName = part.metadata?.filename ?? part.metadata?.fileName ?? part.fileName ?? part.filename ?? part.name ?? part.file?.filename;
+        return { type: "file", source, ...fileName ? { fileName } : {} };
+      }
+    }
+    if (part.type === "audio" || part.type === "input_audio") {
+      const ia = part.input_audio;
+      if (ia?.data) {
+        const mediaType = ia.format === "wav" ? "audio/wav" : "audio/mpeg";
+        return { type: "audio", source: { type: "base64", mediaType, data: ia.data } };
+      }
+      const source = extractBinarySource(part);
+      if (source) return { type: "audio", source };
+    }
     return { type: "text", text: part.text ?? part.content ?? JSON.stringify(part) };
   });
 }
+function extractBinarySource(part) {
+  const src = part.source;
+  if (src?.type === "base64" && src.data) {
+    return { type: "base64", mediaType: src.mediaType ?? src.media_type ?? src.mime_type ?? "application/octet-stream", data: src.data };
+  }
+  if (src?.type === "url" && src.url) {
+    const mediaType2 = src.mediaType ?? src.media_type ?? src.mime_type;
+    return { type: "url", url: src.url, ...mediaType2 ? { mediaType: mediaType2 } : {} };
+  }
+  const mediaType = part.mime_type ?? part.mimeType ?? part.media_type ?? part.mediaType;
+  if (part.source_type === "base64" && part.data) {
+    return { type: "base64", mediaType: mediaType ?? "application/octet-stream", data: part.data };
+  }
+  if (part.source_type === "url" && part.url) {
+    return { type: "url", url: part.url, ...mediaType ? { mediaType } : {} };
+  }
+  const value = part.data ?? part.url ?? part.file?.file_data;
+  if (typeof value === "string" && value) {
+    const match = value.match(/^data:([^;]+);base64,(.+)$/);
+    if (match) return { type: "base64", mediaType: mediaType ?? match[1], data: match[2] };
+    if (/^https?:\/\//.test(value)) return { type: "url", url: value, ...mediaType ? { mediaType } : {} };
+    if (mediaType) return { type: "base64", mediaType, data: value };
+  }
+  return void 0;
+}
 function isZodSchema(obj) {
   return obj != null && typeof obj === "object" && typeof obj.safeParse === "function" && "_def" in obj;
 }
@@ -10738,6 +10949,7 @@ function createProvider(config) {
   }
 }
 
+exports.ASK_USER_TOOL_NAME = ASK_USER_TOOL_NAME;
 exports.AnthropicProvider = AnthropicProvider;
 exports.AzureProvider = AzureProvider;
 exports.BaseProvider = BaseProvider;