@aj-archipelago/cortex 1.3.65 → 1.3.67

package/helper-apps/cortex-autogen2/tools/__init__.py

@@ -5,6 +5,7 @@ Contains various tool modules for agent capabilities.
 
 from .search_tools import web_search, image_search, combined_search, fetch_webpage, collect_task_images
 from .coding_tools import execute_code
+from .google_cse import google_cse_search
 from .azure_blob_tools import upload_file_to_azure_blob
 from .file_tools import list_files_in_work_dir, read_file_from_work_dir, get_file_info, create_file, download_image
 
@@ -15,6 +16,7 @@ __all__ = [
     "fetch_webpage",
     "collect_task_images",
     "execute_code",
+    "google_cse_search",
     "upload_file_to_azure_blob",
     "list_files_in_work_dir",
     "read_file_from_work_dir",

package/helper-apps/cortex-autogen2/tools/azure_blob_tools.py

@@ -8,13 +8,22 @@ import logging
 import mimetypes
 import uuid
 import time
+import hashlib
 from datetime import datetime, timedelta
 from urllib.parse import urlparse, parse_qs
 from azure.storage.blob import BlobServiceClient, generate_blob_sas, BlobSasPermissions, ContentSettings
 from azure.core.exceptions import AzureError, ServiceResponseError
+import requests
 
 logger = logging.getLogger(__name__)
 
+# Ensure correct MIME types for Office files, especially PPT/PPTX, for proper downloads in browsers
+try:
+    mimetypes.add_type("application/vnd.openxmlformats-officedocument.presentationml.presentation", ".pptx", strict=False)
+    mimetypes.add_type("application/vnd.ms-powerpoint", ".ppt", strict=False)
+except Exception:
+    pass
+
 def _validate_sas_url(url: str) -> bool:
     """Private helper to validate an Azure blob SAS URL."""
     try:
@@ -56,6 +65,8 @@ class AzureBlobUploader:
         
         self.connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
         self.container_name = os.getenv("AZURE_BLOB_CONTAINER", "autogentempfiles")
+        # In-memory deduplication cache: sha256 -> blob_name
+        self._sha256_to_blob: dict = {}
         
         if not self.connection_string:
             raise ValueError("AZURE_STORAGE_CONNECTION_STRING environment variable is required")
@@ -136,6 +147,22 @@ class AzureBlobUploader:
         except Exception:
             pass
 
+        # Compute sha256 to deduplicate repeat uploads during same process lifetime
+        sha256_hex = None
+        try:
+            hasher = hashlib.sha256()
+            with open(file_path, "rb") as fh:
+                for chunk in iter(lambda: fh.read(1024 * 1024), b""):
+                    hasher.update(chunk)
+            sha256_hex = hasher.hexdigest()
+            if sha256_hex in self._sha256_to_blob:
+                # Return prior URL for identical content
+                prior_blob = self._sha256_to_blob[sha256_hex]
+                sas_url = self.generate_sas_url(prior_blob)
+                return {"blob_name": prior_blob, "download_url": sas_url, "deduplicated": True}
+        except Exception:
+            sha256_hex = None
+
         # Simple upload; SDK will handle block uploads automatically for large blobs
         with open(file_path, "rb") as data:
             blob_client.upload_blob(
@@ -149,6 +176,11 @@ class AzureBlobUploader:
         sas_url = self.generate_sas_url(normalized_blob_name)
         if not _validate_sas_url(sas_url):
             raise Exception("Generated SAS URL failed validation.")
+        if sha256_hex:
+            try:
+                self._sha256_to_blob[sha256_hex] = normalized_blob_name
+            except Exception:
+                pass
         return {"blob_name": blob_name, "download_url": sas_url}
 
 # Keep a single function for external calls to use the singleton uploader

package/helper-apps/cortex-autogen2/tools/azure_foundry_agents.py

@@ -110,6 +110,32 @@ except Exception:
     _AZURE_SDK_AVAILABLE = False
 
 
+def _normalize_content_to_parts(content: Any) -> List[Dict[str, Any]]:
+    """Convert arbitrary content to Azure Foundry content parts.
+
+    Rules:
+    - If already an array of typed objects ({type:..., ...}), keep as-is.
+    - If it's a dict with 'type', wrap as a single part.
+    - Otherwise stringify to a single text part.
+    """
+    try:
+        # Already properly typed parts
+        if isinstance(content, list) and all(isinstance(p, dict) and isinstance(p.get("type"), str) for p in content):
+            return content
+        # Single typed object
+        if isinstance(content, dict) and isinstance(content.get("type"), str):
+            return [content]
+        # Anything else -> stringify
+        import json as _json
+        if isinstance(content, (dict, list)):
+            text = _json.dumps(content)
+        else:
+            text = str(content) if content is not None else ""
+        return [{"type": "text", "text": text}]
+    except Exception:
+        return [{"type": "text", "text": str(content) if content is not None else ""}]
+
+
 def _convert_to_azure_foundry_messages(
     context: Optional[str],
     examples: Optional[List[Dict[str, Any]]],
@@ -118,26 +144,26 @@ def _convert_to_azure_foundry_messages(
     azure_messages: List[Dict[str, Any]] = []
 
     if context:
-        azure_messages.append({"role": "system", "content": context})
+        azure_messages.append({"role": "system", "content": _normalize_content_to_parts(context)})
 
     if examples:
         for example in examples:
             try:
                 inp = example.get("input", {})
                 out = example.get("output", {})
-                azure_messages.append({"role": inp.get("author", "user"), "content": inp.get("content")})
-                azure_messages.append({"role": out.get("author", "assistant"), "content": out.get("content")})
+                azure_messages.append({"role": inp.get("author", "user"), "content": _normalize_content_to_parts(inp.get("content"))})
+                azure_messages.append({"role": out.get("author", "assistant"), "content": _normalize_content_to_parts(out.get("content"))})
             except Exception:
                 # ignore malformed example
                 continue
 
     for message in messages or []:
-        # Expect message to have 'author' and 'content' keys in Palm-like format,
-        # or 'role' and 'content' already in Azure format.
-        if "role" in message:
-            azure_messages.append({"role": message.get("role"), "content": message.get("content")})
-        else:
-            azure_messages.append({"role": message.get("author"), "content": message.get("content")})
+        role = message.get("role") or message.get("author")
+        # Normalize any unexpected roles to 'user' to satisfy Foundry role schema
+        if role not in ("user", "assistant", "system"):
+            role = "user"
+        content = message.get("content")
+        azure_messages.append({"role": role, "content": _normalize_content_to_parts(content)})
 
     return azure_messages
 
@@ -285,8 +311,9 @@ def call_azure_foundry_agent(
         On failure returns {"status":"error","error": "..."}
     """
     try:
-        # Prefer using the Azure SDK path if available - it handles auth and endpoints robustly.
-        if _AZURE_SDK_AVAILABLE:
+        # Prefer using the Azure SDK path only when explicitly enabled via env var
+        # Set AZURE_FOUNDRY_USE_SDK=true to enable. Default is to use HTTP path.
+        if _AZURE_SDK_AVAILABLE and str(os.getenv("AZURE_FOUNDRY_USE_SDK", "false")).lower() == "true":
             try:
                 # Build credential: prefer explicit service principal creds in env var, else DefaultAzureCredential
                 cred = None
@@ -455,8 +482,17 @@ def call_azure_foundry_agent(
             "stream": bool(parameters.get("stream") if parameters else False),
         }
 
-        # Merge allowed parameter keys into body
+        # Sanitize and merge allowed parameter keys into body
         if parameters:
+            # Make a shallow copy so we can normalize values safely
+            sanitized_params = dict(parameters)
+            # Coerce unsupported response_format values to 'auto'
+            try:
+                rf = sanitized_params.get("response_format")
+                if isinstance(rf, str) and rf.lower() != "auto":
+                    sanitized_params["response_format"] = "auto"
+            except Exception:
+                pass
             allowed_keys = [
                 "tools",
                 "tool_resources",
@@ -472,8 +508,8 @@ def call_azure_foundry_agent(
                 "truncation_strategy",
             ]
             for k in allowed_keys:
-                if k in parameters:
-                    body[k] = parameters[k]
+                if k in sanitized_params:
+                    body[k] = sanitized_params[k]
 
         url = project_url.rstrip("/") + "/threads/runs"
         headers = {"Content-Type": "application/json"}

package/helper-apps/cortex-autogen2/tools/file_tools.py

@@ -209,41 +209,74 @@ async def download_image(url: str, filename: str, work_dir: Optional[str] = None
             "Chrome/125.0.0.0 Safari/537.36"
         )
         session = requests.Session()
-        session.headers.update({"User-Agent": BROWSER_UA})
-
-        with session.get(url, stream=True, timeout=20, allow_redirects=True) as response:
-            response.raise_for_status()
-
-            content_type = (response.headers.get("Content-Type") or "").lower()
-
-            # Peek first few bytes to validate image magic if header is missing/misleading
-            first_chunk = next(response.iter_content(chunk_size=4096), b"")
-
-            def looks_like_image(buf: bytes) -> bool:
-                if not buf or len(buf) < 4:
-                    return False
-                sigs = [
-                    b"\x89PNG\r\n\x1a\n",  # PNG
-                    b"\xff\xd8\xff",        # JPEG
-                    b"GIF87a", b"GIF89a",    # GIF
-                    b"RIFF"                    # WEBP starts with RIFF
-                ]
-                return any(buf.startswith(sig) for sig in sigs)
-
-            if not (content_type.startswith("image/") or looks_like_image(first_chunk)):
-                logger.error(f"❌ URL did not return an image content-type: {content_type} for {url}")
-                return json.dumps({"status": "error", "message": f"Non-image content-type: {content_type}"})
-
-            # Write first chunk then stream the rest
-            with open(file_path, 'wb') as f:
-                if first_chunk:
-                    f.write(first_chunk)
-                for chunk in response.iter_content(chunk_size=8192):
-                    if chunk:
-                        f.write(chunk)
-
-        logger.info(f"✅ Successfully downloaded image from {url} to {file_path}")
-        return json.dumps({"status": "success", "file_path": file_path})
+        session.headers.update({
+            "User-Agent": BROWSER_UA,
+            "Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.9",
+            "Referer": "https://duckduckgo.com/",
+            "Cache-Control": "no-cache",
+        })
+
+        # Attempt to derive an original Wikimedia URL if this is a thumbnail
+        wm_orig = None
+        try:
+            if "upload.wikimedia.org" in url and "/thumb/" in url:
+                parts = url.split("/thumb/")
+                if len(parts) == 2:
+                    tail = parts[1]
+                    segs = tail.split("/")
+                    if len(segs) >= 3:
+                        wm_orig = parts[0] + "/" + segs[0] + "/" + segs[1] + "/" + segs[2]
+        except Exception:
+            wm_orig = None
+
+        candidates = []
+        if wm_orig:
+            candidates.append(wm_orig)
+        candidates.append(url)
+
+        last_err = None
+        for candidate in candidates:
+            try:
+                with session.get(candidate, stream=True, timeout=25, allow_redirects=True) as response:
+                    response.raise_for_status()
+
+                    content_type = (response.headers.get("Content-Type") or "").lower()
+
+                    # Peek first few bytes to validate image magic if header is missing/misleading
+                    first_chunk = next(response.iter_content(chunk_size=4096), b"")
+
+                    def looks_like_image(buf: bytes) -> bool:
+                        if not buf or len(buf) < 4:
+                            return False
+                        sigs = [
+                            b"\x89PNG\r\n\x1a\n",  # PNG
+                            b"\xff\xd8\xff",        # JPEG
+                            b"GIF87a", b"GIF89a",    # GIF
+                            b"RIFF"                    # WEBP starts with RIFF
+                        ]
+                        return any(buf.startswith(sig) for sig in sigs)
+
+                    if not (content_type.startswith("image/") or looks_like_image(first_chunk)):
+                        last_err = f"Non-image content-type: {content_type} for {candidate}"
+                        continue
+
+                    # Write first chunk then stream the rest
+                    with open(file_path, 'wb') as f:
+                        if first_chunk:
+                            f.write(first_chunk)
+                        for chunk in response.iter_content(chunk_size=8192):
+                            if chunk:
+                                f.write(chunk)
+
+                logger.info(f"✅ Successfully downloaded image from {candidate} to {file_path}")
+                return json.dumps({"status": "success", "file_path": file_path})
+            except Exception as e:
+                last_err = str(e)
+                continue
+
+        logger.error(f"❌ Failed to download image after candidates. Last error: {last_err}")
+        return json.dumps({"status": "error", "message": last_err or "download_failed"})
     except Exception as e:
         logger.error(f"❌ Failed to download image from {url}: {e}")
         return json.dumps({"status": "error", "message": str(e)})
@@ -513,33 +546,125 @@ def get_file_tools(executor_work_dir: Optional[str] = None) -> List[FunctionTool
     tools = []
     
     # Create partial functions with work_dir bound
-    def bound_list_files():
-        return asyncio.run(list_files_in_work_dir(executor_work_dir))
+    def _to_text_part(value: str) -> Dict[str, Any]:
+        # Always return OpenAI-typed content
+        return {"type": "text", "text": value if isinstance(value, str) else str(value)}
+
+    def bound_list_files_typed() -> Dict[str, Any]:
+        return _to_text_part(asyncio.run(list_files_in_work_dir(executor_work_dir)))
     
-    def bound_read_file(filename: str, max_length: int = 5000):
-        return asyncio.run(read_file_from_work_dir(filename, executor_work_dir, max_length))
+    def bound_read_file_typed(filename: str, max_length: int = 5000) -> Dict[str, Any]:
+        return _to_text_part(asyncio.run(read_file_from_work_dir(filename, executor_work_dir, max_length)))
     
-    def bound_get_file_info(filename: str):
-        return asyncio.run(get_file_info(filename, executor_work_dir))
+    def bound_get_file_info_typed(filename: str) -> Dict[str, Any]:
+        return _to_text_part(asyncio.run(get_file_info(filename, executor_work_dir)))
     
     # Add tools
     tools.append(FunctionTool(
-        bound_list_files,
+        bound_list_files_typed,
         name="list_files_in_work_dir",
         description="Intelligently discover and categorize all files in the working directory with comprehensive metadata"
     ))
     
     tools.append(FunctionTool(
-        bound_read_file,
+        bound_read_file_typed,
         name="read_file_from_work_dir", 
         description="Intelligently read and analyze any file type with automatic content detection and preview generation"
     ))
     
     tools.append(FunctionTool(
-        bound_get_file_info,
+        bound_get_file_info_typed,
         name="get_file_info",
         description="Get comprehensive metadata and analysis for any file type including permissions and recommendations"
     ))
+
+    # Add a convenience uploader for the newest deliverables
+    async def _upload_recent_deliverables(max_age_minutes: int = 15, max_files: int = 5) -> str:
+        try:
+            from .azure_blob_tools import upload_file_to_azure_blob
+            import time
+            work_dir = executor_work_dir or os.getcwd()
+            now = time.time()
+            deliverable_exts = {".pptx", ".ppt", ".csv", ".png", ".jpg", ".jpeg", ".pdf", ".zip"}
+            candidates: List[str] = []
+            if os.path.isdir(work_dir):
+                for name in os.listdir(work_dir):
+                    path = os.path.join(work_dir, name)
+                    if os.path.isfile(path) and os.path.splitext(name)[1].lower() in deliverable_exts:
+                        try:
+                            mtime = os.path.getmtime(path)
+                            if now - mtime <= max_age_minutes * 60:
+                                candidates.append(path)
+                        except Exception:
+                            continue
+            candidates.sort(key=lambda p: os.path.getmtime(p), reverse=True)
+            uploads = []
+            for p in candidates[:max_files]:
+                try:
+                    up_json = upload_file_to_azure_blob(p)
+                    uploads.append(json.loads(up_json))
+                except Exception as e:
+                    uploads.append({"error": str(e), "file": p})
+            return json.dumps({"uploads": uploads})
+        except Exception as e:
+            return json.dumps({"error": str(e)})
+
+    def bound_upload_recent_deliverables_typed(max_age_minutes: int = 15, max_files: int = 5) -> Dict[str, Any]:
+        return _to_text_part(asyncio.run(_upload_recent_deliverables(max_age_minutes, max_files)))
+
+    tools.append(FunctionTool(
+        bound_upload_recent_deliverables_typed,
+        name="upload_recent_deliverables",
+        description="Upload the newest deliverables from the working directory (scans last N minutes) and return their URLs"
+    ))
+
+    # A suggestion-only tool: list likely deliverables without uploading
+    async def _list_recent_deliverables(max_age_minutes: int = 15, max_files: int = 10, min_size_bytes: int = 1024) -> str:
+        try:
+            import time
+            work_dir = executor_work_dir or os.getcwd()
+            now = time.time()
+            deliverable_exts = {".pptx", ".ppt", ".csv", ".png", ".jpg", ".jpeg", ".pdf", ".zip"}
+            suggestions = []
+            if os.path.isdir(work_dir):
+                for name in os.listdir(work_dir):
+                    path = os.path.join(work_dir, name)
+                    if not os.path.isfile(path):
+                        continue
+                    ext = os.path.splitext(name)[1].lower()
+                    if ext not in deliverable_exts:
+                        continue
+                    try:
+                        size = os.path.getsize(path)
+                        if size < min_size_bytes:
+                            continue
+                        mtime = os.path.getmtime(path)
+                        age_s = now - mtime
+                        if age_s > max_age_minutes * 60:
+                            continue
+                        suggestions.append({
+                            "filename": name,
+                            "absolute_path": path,
+                            "size_bytes": size,
+                            "age_seconds": int(age_s),
+                            "extension": ext,
+                        })
+                    except Exception:
+                        continue
+            # Sort by size desc then recency
+            suggestions.sort(key=lambda x: (x["size_bytes"], -x["age_seconds"]), reverse=True)
+            return json.dumps({"suggestions": suggestions[:max_files]})
+        except Exception as e:
+            return json.dumps({"error": str(e)})
+
+    def bound_list_recent_deliverables_typed(max_age_minutes: int = 15, max_files: int = 10, min_size_bytes: int = 1024) -> Dict[str, Any]:
+        return _to_text_part(asyncio.run(_list_recent_deliverables(max_age_minutes, max_files, min_size_bytes)))
+
+    tools.append(FunctionTool(
+        bound_list_recent_deliverables_typed,
+        name="list_recent_deliverables",
+        description="List likely deliverables (by type, size, recency) without uploading; returns suggestions for human-like selection"
+    ))
     
     logger.info(f"✅ Universal file tools created for work_dir: {executor_work_dir}")
     return tools 

package/helper-apps/cortex-autogen2/tools/google_cse.py

@@ -0,0 +1,117 @@
+"""
+Google Custom Search (CSE) tool.
+
+Provides `google_cse_search` async function that agents can call as a FunctionTool.
+It reads API credentials from environment variables:
+  - GOOGLE_CSE_KEY
+  - GOOGLE_CSE_CX
+
+Parameters mirror the CSE REST API where practical. Returns a JSON string.
+"""
+
+import os
+import json
+import requests
+from typing import Any, Dict, Optional
+
+
+def _get_env_or_error() -> Dict[str, str]:
+    api_key = os.getenv("GOOGLE_CSE_KEY")
+    cx_env = os.getenv("GOOGLE_CSE_CX")
+    if not api_key:
+        raise RuntimeError("GOOGLE_CSE_KEY is not set in the environment variables!")
+    if not cx_env:
+        raise RuntimeError("GOOGLE_CSE_CX is not set in the environment variables!")
+    return {"key": api_key, "cx": cx_env}
+
+
+def _build_params(text: Optional[str], parameters: Optional[Dict[str, Any]], env_cx: str) -> Dict[str, Any]:
+    parameters = parameters or {}
+    # Required
+    q = (parameters.get("q") or text or "")
+    cx = parameters.get("cx") or env_cx
+
+    params: Dict[str, Any] = {
+        "q": q,
+        "cx": cx,
+    }
+
+    # Optional passthroughs
+    if "num" in parameters and parameters["num"] is not None:
+        params["num"] = parameters["num"]
+    if "start" in parameters and parameters["start"] is not None:
+        params["start"] = parameters["start"]
+    if parameters.get("safe"):
+        params["safe"] = parameters["safe"]
+    if parameters.get("dateRestrict"):
+        params["dateRestrict"] = parameters["dateRestrict"]
+    if parameters.get("siteSearch"):
+        params["siteSearch"] = parameters["siteSearch"]
+    if parameters.get("siteSearchFilter"):
+        params["siteSearchFilter"] = parameters["siteSearchFilter"]
+    if parameters.get("searchType"):
+        params["searchType"] = parameters["searchType"]
+    # Image-specific filters
+    if parameters.get("imgSize"):
+        params["imgSize"] = parameters["imgSize"]
+    if parameters.get("imgType"):
+        params["imgType"] = parameters["imgType"]
+    if parameters.get("imgColorType"):
+        params["imgColorType"] = parameters["imgColorType"]
+    if parameters.get("imgDominantColor"):
+        params["imgDominantColor"] = parameters["imgDominantColor"]
+    if parameters.get("imgAspectRatio"):
+        params["imgAspectRatio"] = parameters["imgAspectRatio"]
+    if parameters.get("rights"):
+        params["rights"] = parameters["rights"]
+    if parameters.get("gl"):
+        params["gl"] = parameters["gl"]
+    if parameters.get("hl"):
+        params["hl"] = parameters["hl"]
+    if parameters.get("lr"):
+        params["lr"] = parameters["lr"]
+    if parameters.get("sort"):
+        params["sort"] = parameters["sort"]
+    if parameters.get("exactTerms"):
+        params["exactTerms"] = parameters["exactTerms"]
+    if parameters.get("excludeTerms"):
+        params["excludeTerms"] = parameters["excludeTerms"]
+    if parameters.get("orTerms"):
+        params["orTerms"] = parameters["orTerms"]
+    if parameters.get("fileType"):
+        params["fileType"] = parameters["fileType"]
+
+    return params
+
+
+async def google_cse_search(
+    text: Optional[str] = None,
+    parameters: Optional[Dict[str, Any]] = None,
+) -> str:
+    """
+    Perform a Google Custom Search.
+
+    Args:
+        text: query text (used if `parameters.q` not provided)
+        parameters: optional extra parameters per CSE API (e.g., num, start, safe, dateRestrict, etc.)
+
+    Returns:
+        JSON string of the raw CSE API response.
+    """
+    try:
+        creds = _get_env_or_error()
+        api_key = creds["key"]
+        cx = creds["cx"]
+
+        params = _build_params(text, parameters, cx)
+        params["key"] = api_key
+
+        url = "https://www.googleapis.com/customsearch/v1"
+        resp = requests.get(url, params=params, timeout=20)
+        resp.raise_for_status()
+        data = resp.json()
+        return json.dumps(data)
+    except Exception as exc:
+        return json.dumps({"error": f"google_cse_search failed: {str(exc)}"})
+
+