@aj-archipelago/cortex 1.3.66 → 1.3.67

package/helper-apps/cortex-autogen2/Dockerfile

@@ -1,31 +1,98 @@
-# Use the official Azure Functions Python base image
+# ────────────────────────────────────────────────────────────────
+# BASE IMAGE  (Python 3.11, Azure Functions Runtime v4)
+# ────────────────────────────────────────────────────────────────
 FROM mcr.microsoft.com/azure-functions/python:4-python3.11
 
-# Set the working directory
-WORKDIR /home/site/wwwroot
+# ------------------------------------------------------------------------------
+# 1. Azure Functions environment variables
+# ------------------------------------------------------------------------------
+ENV AzureWebJobsScriptRoot=/home/site/wwwroot \
+    AzureFunctionsJobHost__Logging__Console__IsEnabled=true \
+    WEBSITES_INCLUDE_CLOUD_CERTS=true \
+    FUNCTIONS_WORKER_RUNTIME=python \
+    AzureWebJobsFeatureFlags=EnableWorkerIndexing
+
+# ------------------------------------------------------------------------------
+# 2. System-level packages required by many of the new Python deps
+# ------------------------------------------------------------------------------
 
-# Install system dependencies
-RUN apt-get update && apt-get install -y \
-    build-essential \
-    curl \
-    git \
-    libgirepository1.0-dev \
-    pkg-config \
-    libfreetype6-dev \
-    libpng-dev \
-    fontconfig \
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive \
+    apt-get install -y --no-install-recommends \
+        # ─ Fonts & Playwright basics (including Arabic/RTL support)
+        fonts-liberation fonts-noto fontconfig \
+        fonts-noto-core fonts-noto-ui-core \
+        fonts-noto-color-emoji fonts-noto-cjk \
+        fonts-dejavu fonts-dejavu-core fonts-dejavu-extra \
+        fonts-freefont-ttf fonts-liberation2 \
+        # ─ Arabic fonts specifically for matplotlib/reportlab
+        fonts-arabeyes fonts-farsiweb fonts-kacst fonts-kacst-one \
+        fonts-hosny-amiri fonts-sil-scheherazade fonts-sil-lateef \
+        # ─ Build chain for packages that still compile C/C++
+        build-essential gcc g++ make \
+        # ─ libmagic for `python-magic`
+        libmagic1 libmagic-dev \
+        # ─ FFmpeg for `moviepy`, `imageio[ffmpeg]`, `pydub`
+        ffmpeg \
+        # ─ HEIF/HEIC support for `pyheif`
+        libde265-dev libheif-dev \
+        # ─ Audio support for `librosa` / `soundfile`
+        libsndfile1 \
+        # ─ Computer-vision helpers sometimes needed by OpenCV
+        libsm6 libxext6 libglib2.0-0 \
+        # ─ HDF5 stack for `h5py`
+        libhdf5-serial-dev hdf5-tools \
+        # ─ Cairo / Pango / GDK-PixBuf for SVG->PNG rendering (CairoSVG optional deps)
+        libcairo2 libcairo2-dev \
+        libpango-1.0-0 libpangoft2-1.0-0 libpangocairo-1.0-0 \
+        libgdk-pixbuf-2.0-0 libgdk-pixbuf2.0-bin \
+        # ─ GDAL for GIS formats (and Python bindings)
+        gdal-bin libgdal-dev python3-gdal \
+        # ─ 7-Zip & RAR extractors (for patool / rarfile fall-back)
+        unrar-free \
+        # ─ Clean-up
+    && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
-RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg \
-    && mv microsoft.gpg /etc/apt/trusted.gpg.d/microsoft.gpg \
-    && sh -c 'echo "deb [arch=amd64,arm64,armhf] https://packages.microsoft.com/repos/microsoft-debian-bullseye-prod bullseye main" > /etc/apt/sources.list.d/azure-cli.list' \
-    && apt-get update \
-    && apt-get install -y azure-functions-core-tools
 
-# Copy requirements and install Python dependencies
+# Allow GDAL Python wheels to find headers
+ENV CPLUS_INCLUDE_PATH=/usr/include/gdal
+ENV C_INCLUDE_PATH=/usr/include/gdal
+
+# ------------------------------------------------------------------------------
+# 3. Python dependencies
+#    – We copy only the files that affect dependency resolution first
+#      (leverages Docker layer cache).
+# ------------------------------------------------------------------------------
+WORKDIR /tmp
+
+# Copy requirements for pip-based install
 COPY requirements.txt .
+
+# Use a pip version compatible with packages that have legacy metadata (e.g., textract 1.6.5)
+RUN python -m pip install --upgrade 'pip<24.1' setuptools wheel
+
+# Install all Python packages — no cache -> smaller image
 RUN pip install --no-cache-dir -r requirements.txt
 
-# Copy application code
+# Rebuild font cache to ensure all Arabic fonts are detected
+RUN fc-cache -fv
+
+# ------------------------------------------------------------------------------
+# 4. Playwright browsers (chromium only to keep image light)
+# ------------------------------------------------------------------------------
+RUN playwright install --with-deps chromium
+
+# ------------------------------------------------------------------------------
+# 5. Copy your Azure Functions source code
+# ------------------------------------------------------------------------------
+WORKDIR /home/site/wwwroot
 COPY . .
-CMD ["func", "start", "--port", "80", "--verbose"]
 
+# Ports: Azure Functions host inside the container listens on 80 by default.
+# (You normally don’t need EXPOSE for Azure App Service, but uncomment if you
+# run the image locally.)
+# EXPOSE 80
+
+# ------------------------------------------------------------------------------
+# 6. The base image already contains the correct ENTRYPOINT / CMD
+#    to launch the Functions host, so nothing to add here.
+# ------------------------------------------------------------------------------

package/helper-apps/cortex-autogen2/docker-compose.yml

@@ -7,14 +7,21 @@ services:
         - linux/amd64
     ports:
       - "7071:80"
+    env_file:
+      - .env
     environment:
-      - CORTEX_API_BASE_URL=http://host.docker.internal:4000/v1
-      - REDIS_CONNECTION_STRING=redis://host.docker.internal:6379
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:80/api/health"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-      start_period: 60s
+      - FUNCTIONS_WORKER_RUNTIME=python
+      - AzureWebJobsFeatureFlags=EnableWorkerIndexing
+      - AZURE_STORAGE_CONNECTION_STRING=${AZURE_STORAGE_CONNECTION_STRING}
+      - AzureWebJobsStorage=${AZURE_STORAGE_CONNECTION_STRING}
+      - QUEUE_NAME=${AZURE_QUEUE_NAME:-${QUEUE_NAME}}
+      - CORTEX_API_BASE_URL=${CORTEX_API_BASE_URL:-http://host.docker.internal:4000/v1}
+      - REDIS_CONNECTION_STRING=${REDIS_CONNECTION_STRING:-redis://host.docker.internal:6379}
+    # healthcheck:
+    #   test: ["CMD", "curl", "-f", "http://localhost:80/api/health"]
+    #   interval: 30s
+    #   timeout: 10s
+    #   retries: 3
+    #   start_period: 60s
     restart: unless-stopped
     network_mode: host

package/helper-apps/cortex-autogen2/host.json

@@ -11,5 +11,10 @@
   "extensionBundle": {
     "id": "Microsoft.Azure.Functions.ExtensionBundle",
     "version": "[4.*, 5.0.0)"
+  },
+  "extensions": {
+    "queues": {
+      "maxPollingInterval": "00:00:01"
+    }
   }
 }

package/helper-apps/cortex-autogen2/pyproject.toml

@@ -1,36 +1,93 @@
 [tool.poetry]
-name = "cortex-autogen2"
-version = "0.1.0"
+name        = "cortex-autogen2"
+version     = "0.1.0"
 description = "Multi-agent coding assistant using AutoGen"
-authors = ["Your Name <your.email@example.com>"]
-readme = "README.md"
-packages = [{include = "cortex_autogen2", from = "src"}]
+authors     = ["Your Name <your.email@example.com>"]
+readme      = "README.md"
+packages    = [{ include = "cortex_autogen2", from = "src" }]
 
+# ─────────────────────────────────────────────────
+# Core runtime (always installed)
+# ─────────────────────────────────────────────────
 [tool.poetry.dependencies]
 python = ">=3.11,<3.13"
-autogen-agentchat = {extras = ["openai"], version = "^0.6.4"}
-python-dotenv = "^1.0.1"
-requests = "^2.32.3"
-redis = "^5.0.4"
+
+# File-handling essentials
+Pillow            = "^11.0.0"
+python-magic      = "^0.4.27"
+chardet           = "^5.2.0"
+PyPDF2            = "^3.0.1"
+pdfminer.six      = "^20221105"
+docx2txt          = ">=0.9,<1.0"
+openpyxl          = "^3.1.2"
+xlrd              = "^2.0.1"
+python-pptx       = "^0.6.23"
+odfpy             = "^1.4.1"
+
+# Infra / networking / AI you already had
+autogen-agentchat = { extras = ["openai"], version = "^0.7.4" }
+autogen-ext       = { extras = ["docker"], version = "^0.7.4" }
+openai            = "^1.97.1"
+tiktoken          = "^0.9.0"
+python-dotenv     = "^1.0.1"
+requests          = "^2.32.3"
+redis             = "^5.0.4"
 azure-storage-queue = "^12.10.1"
-azure-storage-blob = "^12.19.0"
-sqlalchemy = "^2.0.30"
-Pillow = "^11.0.0"
-pymysql = "^1.1.1"
-playwright = "^1.54.0"
-markitdown = "^0.1.2"
-docker = "^7.1.0"
-autogen-ext = {extras = ["docker"], version = "^0.6.4"}
-openai = "^1.97.1"
-tiktoken = "^0.9.0"
-aiofiles = "^24.1.0"
-pandas = "^2.3.1"
-matplotlib = "^3.10.3"
-aiohttp = "^3.12.14"
+azure-storage-blob  = "^12.19.0"
+sqlalchemy        = "^2.0.30"
+pymysql           = "^1.1.1"
+docker            = "^7.1.0"
+playwright        = "^1.54.0"
+markitdown        = "^0.1.2"
+aiofiles          = "^24.1.0"
+aiohttp           = "^3.12.14"
+pandas            = "^2.3.1"
+matplotlib        = "^3.10.3"
+lxml              = "^5.3.0"
+reportlab         = "^4.2.5"
+fpdf2             = "^2.7.9"
+
+# ─────────────────────────────────────────────────
+# Optional groups (install with: poetry install --with media,archives,science …)
+# ─────────────────────────────────────────────────
+
+[tool.poetry.group.media.dependencies]  # images / audio / video
+opencv-python    = "^4.9.0"
+imageio          = { version = "^2.34.0", extras = ["ffmpeg"] }
+pyheif           = "^0.6.3"
+moviepy          = "^1.0.3"
+librosa          = "^0.10.1"
+soundfile        = "^0.12.1"
+pydub            = "^0.25.1"
+
+[tool.poetry.group.archives.dependencies]
+patool   = "^1.12"
+py7zr    = "^0.21.0"
+rarfile  = "^4.0"
+
+[tool.poetry.group.science.dependencies]
+tabula-py = "^2.8.1"
+h5py      = "^3.10.0"
+netCDF4   = "^1.6.5"
+ezdxf     = "^1.1.1"
+
+[tool.poetry.group.comm.dependencies]  # email / web / e-books
+eml-parser       = "^1.15.0"
+msg-parser       = ">=1.2.0,<2.0"
+beautifulsoup4   = "^4.12.3"
+ebooklib         = "^0.18"
+
+[tool.poetry.group.extras.dependencies]  # seldom-needed odds & ends
+imageio-ffmpeg = "^0.4.9"
+python-lzf     = "^0.2.4"
 
+# ─────────────────────────────────────────────────
+# Development-only deps
+# ─────────────────────────────────────────────────
 [tool.poetry.group.dev.dependencies]
 ipykernel = "^6.29.5"
 
+# ─────────────────────────────────────────────────
 [build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api" 
+requires      = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

package/helper-apps/cortex-autogen2/requirements.txt

@@ -1,20 +1,90 @@
+# -------------------------------------------------------------------
+# ESSENTIAL – office docs, PDFs, images
+# -------------------------------------------------------------------
+Pillow>=11.0.0
+python-magic>=0.4.27
+chardet>=5.2.0
+PyPDF2>=3.0.1
+pdfminer.six>=20221105
+docx2txt>=0.9,<1.0
+openpyxl>=3.1.2
+xlrd>=2.0.1
+python-pptx>=0.6.23
+odfpy>=1.4.1
+
+# -------------------------------------------------------------------
+# MEDIA – images / audio / video
+# -------------------------------------------------------------------
+opencv-python>=4.9.0
+imageio[ffmpeg]>=2.34.0
+CairoSVG>=2.7.1
+pyheif>=0.6.3
+moviepy>=1.0.3
+librosa>=0.10.1
+soundfile>=0.12.1
+pydub>=0.25.1
+
+# -------------------------------------------------------------------
+# ARCHIVES
+# -------------------------------------------------------------------
+patool>=1.12
+py7zr>=0.21.0
+rarfile>=4.0
+
+# -------------------------------------------------------------------
+# DATA & SCIENTIFIC
+# -------------------------------------------------------------------
+pandas>=2.2.0
+tabula-py>=2.8.1
+h5py>=3.10.0
+netCDF4>=1.6.5
+ezdxf>=1.1.1
+seaborn>=0.13.0
+
+# -------------------------------------------------------------------
+# ARABIC & RTL TEXT SUPPORT
+# -------------------------------------------------------------------
+arabic-reshaper>=3.0.0
+python-bidi>=0.4.2
+
+# -------------------------------------------------------------------
+# EMAIL / WEB / E-BOOK
+# -------------------------------------------------------------------
+eml-parser>=1.15.0
+msg-parser>=1.2.0,<2.0
+beautifulsoup4>=4.12.3
+ebooklib>=0.18
+
+# -------------------------------------------------------------------
+# CLOUD / INFRASTRUCTURE
+# -------------------------------------------------------------------
 azure-functions>=1.20.0
-autogen-agentchat[openai]>=0.6.4
-python-dotenv>=1.0.1
-requests>=2.32.3
-redis>=5.0.4
-azure-storage-queue>=12.10.1
 azure-storage-blob>=12.19.0
+azure-storage-queue>=12.10.1
+redis>=5.0.4
 sqlalchemy>=2.0.30
-Pillow>=11.0.0
 pymysql>=1.1.1
-playwright>=1.54.0
-markitdown>=0.1.2
-docker>=7.1.0
-autogen-ext[docker]>=0.6.4
+requests>=2.32.3
+aiohttp>=3.12.14
+aiofiles>=24.1.0
+
+# -------------------------------------------------------------------
+# AI / AUTOMATION
+# -------------------------------------------------------------------
 openai>=1.97.1
 tiktoken>=0.9.0
-aiofiles>=24.1.0
-aiohttp>=3.12.14 
-pandas
-matplotlib 
+autogen-agentchat[openai]>=0.7.4
+autogen-ext[docker]>=0.7.4
+docker>=7.1.0
+playwright>=1.54.0
+markitdown>=0.1.2
+python-dotenv>=1.0.1
+reportlab>=4.2.5
+fpdf2>=2.7.9
+matplotlib>=3.9.0
+
+# -------------------------------------------------------------------
+# OPTIONAL EXTRAS
+# -------------------------------------------------------------------
+imageio-ffmpeg>=0.4.9
+python-lzf>=0.2.4

package/helper-apps/cortex-autogen2/services/redis_publisher.py

@@ -1,7 +1,9 @@
 import redis
 import json
 import logging
-from typing import Dict, Any, Optional
+import asyncio
+import time
+from typing import Dict, Any, Optional, List
 
 import os
 
@@ -52,13 +54,31 @@ def connect_redis() -> bool:
                 return False
         return False
 
+_last_logged_progress: Dict[str, Any] = {}
+
 def publish_request_progress(data: Dict[str, Any]) -> bool:
-    """Publish progress data to Redis channel - matches working version pattern"""
+    """Publish progress data to Redis channel with minimal logging (only when message changes)."""
     if connect_redis():
         try:
             message = json.dumps(data)
             result = redis_client.publish(os.getenv("REDIS_CHANNEL"), message)
-            logger.info(f"Published progress update for request {data.get('requestId')}: progress={data.get('progress')}, subscribers={result}")
+            try:
+                rid = data.get('requestId')
+                info = data.get('info')
+                pct = data.get('progress')
+                prev = _last_logged_progress.get(rid)
+                # Log only if info or integer progress changed
+                pct_bucket = None
+                try:
+                    pct_bucket = int(float(pct) * 100)
+                except Exception:
+                    pct_bucket = None
+                if not prev or prev.get('info') != info or prev.get('pct_bucket') != pct_bucket:
+                    _last_logged_progress[rid] = {'info': info, 'pct_bucket': pct_bucket}
+                    logger.info(f"Published progress update for request {rid}: progress={pct}, subscribers={result}")
+            except Exception:
+                # Safe fallback if logging diff fails
+                logger.debug("Progress publish logged without diff due to exception")
             return True
         except Exception as e:
             logger.error(f"Error publishing message to Redis: {e}")
@@ -72,6 +92,23 @@ class RedisPublisher:
     
     def __init__(self):
         self.connected = False
+        # Heartbeat + transient caching
+        self._heartbeat_task: Optional[asyncio.Task] = None
+        try:
+            # Clamp to at most 1.0s to ensure the UI gets updates every second
+            interval = float(os.getenv("PROGRESS_HEARTBEAT_INTERVAL", "1.0"))
+            if interval > 1.0:
+                interval = 1.0
+            if interval <= 0:
+                interval = 1.0
+            self._interval_seconds = interval
+        except Exception:
+            self._interval_seconds = 1.0
+        # We cache only summarized progress strings (emoji sentence) with progress float
+        self._transient_latest: Dict[str, Dict[str, Any]] = {}
+        self._transient_all: Dict[str, List[Dict[str, Any]]] = {}
+        self._finalized: Dict[str, bool] = {}
+        self._lock = asyncio.Lock()
         
     async def connect(self):
         """Initialize Redis connection"""
@@ -81,6 +118,13 @@ class RedisPublisher:
         else:
             logger.warning("Failed to connect to Redis")
             logger.warning("Redis progress publishing will be disabled")
+        # Start heartbeat loop once per process
+        if self._heartbeat_task is None:
+            try:
+                self._heartbeat_task = asyncio.create_task(self._heartbeat_loop())
+                logger.info("Started Redis progress heartbeat task")
+            except Exception as e:
+                logger.warning(f"Failed to start heartbeat task: {e}")
     
     def publish_request_progress(self, data: Dict[str, Any]) -> bool:
         """Publish progress data to Redis channel"""
@@ -99,6 +143,68 @@ class RedisPublisher:
             message_data["data"] = data
             
         return self.publish_request_progress(message_data)
+
+    async def set_transient_update(self, request_id: str, progress: float, info: str) -> None:
+        """Cache the latest summarized transient progress (short sentence with emoji).
+        Heartbeat will re-publish this every second until final. No raw chat content here."""
+        try:
+            async with self._lock:
+                # Skip if already finalized
+                if self._finalized.get(request_id):
+                    return
+                self._transient_latest[request_id] = {"progress": progress, "info": info, "ts": time.time()}
+                lst = self._transient_all.get(request_id)
+                if lst is None:
+                    lst = []
+                    self._transient_all[request_id] = lst
+                lst.append({"progress": progress, "info": info, "ts": time.time()})
+                # Avoid unbounded growth
+                if len(lst) > 200:
+                    del lst[: len(lst) - 200]
+        except Exception as e:
+            logger.warning(f"set_transient_update error for {request_id}: {e}")
+
+    async def mark_final(self, request_id: str) -> None:
+        """Mark a request as finalized to stop transient heartbeat for it."""
+        try:
+            async with self._lock:
+                self._finalized[request_id] = True
+                # Optionally clear cached transient
+                if request_id in self._transient_latest:
+                    del self._transient_latest[request_id]
+        except Exception as e:
+            logger.warning(f"mark_final error for {request_id}: {e}")
+
+    async def _heartbeat_loop(self):
+        """Background loop that emits latest transient updates every interval."""
+        try:
+            while True:
+                try:
+                    # Snapshot under lock
+                    async with self._lock:
+                        items = [
+                            (rid, payload)
+                            for rid, payload in self._transient_latest.items()
+                            if not self._finalized.get(rid)
+                        ]
+                    if items:
+                        for rid, payload in items:
+                            try:
+                                message_data = {
+                                    "requestId": rid,
+                                    "progress": float(payload.get("progress", 0.0)),
+                                    "info": str(payload.get("info", ""))
+                                }
+                                self.publish_request_progress(message_data)
+                            except Exception as pub_err:
+                                logger.debug(f"Heartbeat publish error for {rid}: {pub_err}")
+                except Exception as loop_err:
+                    logger.debug(f"Heartbeat loop iteration error: {loop_err}")
+                await asyncio.sleep(self._interval_seconds)
+        except asyncio.CancelledError:
+            logger.info("Redis progress heartbeat task cancelled")
+        except Exception as e:
+            logger.warning(f"Heartbeat loop terminated unexpectedly: {e}")
     
     def store_final_result(self, request_id: str, result_data: Dict[str, Any], expiry_seconds: int = 3600) -> bool:
         """Store final result in Redis key for retrieval"""
@@ -129,6 +235,18 @@ class RedisPublisher:
     async def close(self):
         """Close Redis connection gracefully"""
         global redis_client
+        # Stop heartbeat
+        if self._heartbeat_task is not None:
+            try:
+                self._heartbeat_task.cancel()
+                try:
+                    await self._heartbeat_task
+                except asyncio.CancelledError:
+                    pass
+            except Exception as e:
+                logger.debug(f"Error cancelling heartbeat task: {e}")
+            finally:
+                self._heartbeat_task = None
         if redis_client:
             try:
                 # Don't actually close the connection in non-continuous mode
@@ -150,4 +268,12 @@ async def get_redis_publisher() -> RedisPublisher:
     if _redis_publisher is None:
         _redis_publisher = RedisPublisher()
         await _redis_publisher.connect()
+    else:
+        # Ensure connectivity and heartbeat are active for new tasks
+        try:
+            if (not getattr(_redis_publisher, 'connected', False)) or getattr(_redis_publisher, '_heartbeat_task', None) is None:
+                await _redis_publisher.connect()
+        except Exception:
+            # Best-effort reconnect
+            await _redis_publisher.connect()
     return _redis_publisher