@aj-archipelago/cortex 1.3.49 → 1.3.51

package/helper-apps/cortex-browser/function_app.py

@@ -1,181 +1,708 @@
-import azure.functions as func
-import logging
-import json
-from playwright.sync_api import sync_playwright
-import trafilatura
-import base64
-
-app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS)
-
-def scrape_and_screenshot(url: str, should_screenshot: bool = True) -> dict:
-    """Scrapes text and takes a screenshot of a given URL, attempting to reject cookies."""
-    screenshot_bytes = None
-    html_content = None
-    extracted_text = None
-
-    try:
-        with sync_playwright() as p:
-            browser = p.chromium.launch(headless=True)
-            try:
-                context = browser.new_context()
-                page = context.new_page()
-                page.goto(url, wait_until='load', timeout=60000) # Increased timeout
-
-                # --- Attempt to reject cookies ---
-                # Add more selectors here if needed for different sites
-                reject_selectors = [
-                    "button:has-text('Reject All')",
-                    "button:has-text('Decline')",
-                    "button:has-text('Only necessary')",
-                    "button:has-text('Tümünü Reddet')", # From your example
-                    "button:has-text('Reject')",
-                    "[aria-label*='Reject']", # Common aria labels
-                    "[id*='reject']",
-                    "[class*='reject']",
-                    # Add more specific selectors based on common banner frameworks if known
-                ]
-
-                cookie_banner_found = False
-                for selector in reject_selectors:
-                    try:
-                        # Wait briefly for the banner element to appear
-                        reject_button = page.locator(selector).first
-                        if reject_button.is_visible(timeout=2000): # Wait up to 2 seconds
-                            logging.info(f"Found potential cookie reject button with selector: {selector}")
-                            reject_button.click(timeout=5000) # Click with a timeout
-                            logging.info("Clicked cookie reject button.")
-                            # Wait a tiny bit for the banner to disappear/page to settle
-                            page.wait_for_timeout(500)
-                            cookie_banner_found = True
-                            break # Stop searching once one is clicked
-                    except Exception as e:
-                        # Ignore timeout errors if the element doesn't appear or other exceptions
-                        # logging.debug(f"Cookie reject selector '{selector}' not found or failed: {e}")
-                        pass # Try the next selector
-
-                if not cookie_banner_found:
-                     logging.info("No common cookie reject button found or clicked.")
-                # ---------------------------------
-
-                html_content = page.content()
-                # Take FULL page screenshot before closing
-                if should_screenshot:
-                    screenshot_bytes = page.screenshot(full_page=True) # Added full_page=True
-            finally:
-                browser.close()
-    except Exception as e:
-        logging.error(f"Playwright error accessing {url}: {e}")
-        return {"url": url, "error": f"Playwright error: {e}"}
-
-    if html_content:
-        try:
-            extracted_text = trafilatura.extract(html_content, include_comments=False)
-        except Exception as e:
-            logging.error(f"Trafilatura error processing {url}: {e}")
-            # Still return screenshot if Playwright succeeded
-            extracted_text = f"Trafilatura extraction failed: {e}"
-
-    screenshot_base64 = base64.b64encode(screenshot_bytes).decode('utf-8') if screenshot_bytes else None
-
-    response_data = {
-        "url": url,
-        "text": extracted_text or "",
-    }
-    if screenshot_base64:
-        response_data["screenshot_base64"] = screenshot_base64
-
-    return response_data
-
-@app.route(route="scrape") # Changed route name
-def http_scrape_trigger(req: func.HttpRequest) -> func.HttpResponse:
-    logging.info('Python HTTP scrape trigger function processed a request.')
-
-    url = None
-    take_screenshot = True # Default value
-
-    # 1. Try getting parameters from query string first
-    try:
-        url = req.params.get('url')
-        if url:
-            logging.info(f"Found URL in query parameters: {url}")
-            # Handle take_screenshot from query params
-            ss_param = req.params.get('take_screenshot', 'true') # Query params are strings
-            take_screenshot = ss_param.lower() != 'false'
-        else:
-             logging.info("URL not found in query parameters.")
-    except Exception as e:
-        # This shouldn't generally happen with req.params, but good practice
-        logging.warning(f"Error reading query parameters: {e}")
-        url = None # Ensure url is None if error occurs here
-
-    # 2. If URL not found in query, try getting from JSON body
-    if not url:
-        logging.info("Attempting to read URL from JSON body.")
-        try:
-            req_body = req.get_json()
-            if req_body:
-                url = req_body.get('url')
-                if url:
-                    logging.info(f"Found URL in JSON body: {url}")
-                    # Handle take_screenshot from JSON body
-                    ss_param = req_body.get('take_screenshot', True)
-                    if isinstance(ss_param, str):
-                        take_screenshot = ss_param.lower() != 'false'
-                    else:
-                        take_screenshot = bool(ss_param) # Convert other types
-                    logging.info(f"Screenshot parameter from JSON: {take_screenshot}")
-                else:
-                    logging.info("URL key not found in JSON body.")
-            else:
-                logging.info("JSON body is empty.")
-        except ValueError:
-            logging.info("Request body is not valid JSON or missing.")
-            # url remains None
-        except Exception as e:
-            logging.warning(f"Error reading JSON body: {e}")
-            url = None # Ensure url is None if error occurs here
-
-    # 3. Process the request if URL was found
-    if url:
-        try:
-            # Validate URL basic structure (optional but recommended)
-            if not url.startswith(('http://', 'https://')):
-                 raise ValueError("Invalid URL format. Must start with http:// or https://")
-
-            result_data = scrape_and_screenshot(url, should_screenshot=take_screenshot) # Pass the flag
-            return func.HttpResponse(
-                json.dumps(result_data),
-                mimetype="application/json",
-                status_code=200
-            )
-        except ValueError as ve:
-             logging.error(f"Invalid URL provided: {ve}")
-             return func.HttpResponse(
-                  json.dumps({"error": str(ve)}),
-                  mimetype="application/json",
-                  status_code=400
-             )
-        except Exception as e:
-            logging.error(f"Error processing scrape request for {url}: {e}")
-            return func.HttpResponse(
-                 json.dumps({"error": f"An internal error occurred: {e}"}),
-                 mimetype="application/json",
-                 status_code=500
-            )
-    else:
-        logging.warning("URL not provided in request body or query string.")
-        return func.HttpResponse(
-             json.dumps({"error": "Please pass a 'url' in the JSON request body or query string"}),
-             mimetype="application/json",
-             status_code=400
-        )
-
-# Keep this if you might have other triggers, otherwise it can be removed
-# if the scrape trigger is the only one.
-# Example of another potential trigger (e.g., timer)
-# @app.timer_trigger(schedule="0 */5 * * * *", arg_name="myTimer", run_on_startup=True,
-#                    use_monitor=False)
-# def timer_trigger_handler(myTimer: func.TimerRequest) -> None:
-#     if myTimer.past_due:
-#         logging.info('The timer is past due!')
-#     logging.info('Python timer trigger function executed.')
+import azure.functions as func
+import logging
+import json
+import base64
+import time
+import os
+import tempfile
+import re
+import asyncio
+from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeoutError, Error as PlaywrightError
+import trafilatura
+from typing import Union, Dict, Any, Tuple, Optional
+import shutil
+from aiohttp import web # Added for local server
+
+app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS)
+
+# Get global timeout from environment variable, default to 20
+timeout_str = os.environ.get("GLOBAL_TIMEOUT_SECONDS", "20")
+try:
+    GLOBAL_TIMEOUT_SECONDS = int(timeout_str)
+except ValueError:
+    logging.warning(f"Invalid GLOBAL_TIMEOUT_SECONDS environment variable '{timeout_str}'. Defaulting to 20.")
+    GLOBAL_TIMEOUT_SECONDS = 20
+
+MIN_TIME_FOR_STEP = 0.5  # Minimum time to allow for any single step if budget is very low
+BUFFER_FOR_CLEANUP_AND_PROCESSING = 2  # Reserve N seconds for trafilatura, screenshot, and cleanup
+SCREENSHOT_DIR_PLAYWRIGHT = "downloaded_files/playwright_screenshots"
+
+def get_remaining_time(start_time: float, budget_for_step: Union[float, None] = None) -> float:
+    """Calculates remaining time from the global timeout."""
+    elapsed_time = time.time() - start_time
+    remaining_global_time = GLOBAL_TIMEOUT_SECONDS - elapsed_time
+    if budget_for_step is not None:
+        safe_budget = min(budget_for_step, remaining_global_time - BUFFER_FOR_CLEANUP_AND_PROCESSING)
+        return max(MIN_TIME_FOR_STEP, safe_budget)
+    return max(0, remaining_global_time - BUFFER_FOR_CLEANUP_AND_PROCESSING)
+
+def sanitize_filename(url_or_name: str) -> str:
+    """Sanitizes a string to be a valid filename."""
+    # Remove http(s)://
+    name = re.sub(r'^https?://', '', url_or_name)
+    # Replace non-alphanumeric characters (except . - _) with underscores
+    name = re.sub(r'[^a-zA-Z0-9._-]', '_', name)
+    # Truncate if too long
+    return name[:100] 
+
+# --- Task for Text Extraction ---
+async def _task_extract_text(page, start_time: float) -> Tuple[Optional[str], Optional[str]]:
+    extracted_text = None
+    html_content = None
+    error_message = None
+    timed_out_early_task = False
+
+    try:
+        logging.info("[Text Task] Attempting primary text extraction with body.inner_text()...")
+        primary_inner_text_budget_s = get_remaining_time(start_time, budget_for_step=7)
+        if primary_inner_text_budget_s < MIN_TIME_FOR_STEP:
+            logging.warning(f"[Text Task] Skipping primary body.inner_text() due to insufficient time ({primary_inner_text_budget_s:.2f}s).")
+        else:
+            logging.info(f"[Text Task] Allocating {primary_inner_text_budget_s:.2f}s for primary body.inner_text().")
+            try:
+                body_text_pl = await asyncio.wait_for(page.locator('body').inner_text(timeout=primary_inner_text_budget_s * 1000), timeout=primary_inner_text_budget_s + 0.5)
+                if body_text_pl:
+                    extracted_text = body_text_pl.strip()
+                    logging.info(f"[Text Task] Captured primary text via body.inner_text(), length: {len(extracted_text)}.")
+                else:
+                    logging.warning("[Text Task] Primary body.inner_text() was empty.")
+            except asyncio.TimeoutError:
+                logging.warning(f"[Text Task] Timeout ({primary_inner_text_budget_s:.2f}s) during primary body.inner_text().")
+                timed_out_early_task = True
+            except Exception as e_primary_body_text:
+                logging.warning(f"[Text Task] Error during primary body.inner_text(): {e_primary_body_text}.")
+                error_message = f"Error during inner_text: {e_primary_body_text}"
+
+        if not extracted_text and not timed_out_early_task:
+            logging.info("[Text Task] Attempting to fetch page.content() for Trafilatura fallback.")
+            page_content_budget_s = get_remaining_time(start_time, budget_for_step=15)
+            if page_content_budget_s < MIN_TIME_FOR_STEP * 2:
+                logging.warning(f"[Text Task] Skipping page.content() fetch due to insufficient time ({page_content_budget_s:.2f}s).")
+            else:
+                logging.info(f"[Text Task] Allocating {page_content_budget_s:.2f}s for page.content().")
+                try:
+                    html_content = await asyncio.wait_for(page.content(), timeout=page_content_budget_s)
+                    if html_content:
+                        logging.info(f"[Text Task] Successfully fetched page.content(), length: {len(html_content)}.")
+                    else:
+                        logging.warning("[Text Task] page.content() returned None or empty.")
+                        html_content = None
+                except asyncio.TimeoutError:
+                    logging.warning(f"[Text Task] Timeout ({page_content_budget_s:.2f}s) while getting page.content(). html_content will be None.")
+                    html_content = None
+                    timed_out_early_task = True
+                except Exception as e_final_ps:
+                    logging.warning(f"[Text Task] Could not get final page.content(): {e_final_ps}")
+                    if not error_message: error_message = f"Failed to get page.content(): {e_final_ps}"
+                    html_content = None
+        elif not timed_out_early_task:
+             logging.info("[Text Task] Skipping page.content() fetch as inner_text succeeded or timed out.")
+
+        if html_content and not extracted_text and not timed_out_early_task:
+            try:
+                logging.info("[Text Task] Attempting Trafilatura text extraction as fallback...")
+                trafilatura_text = trafilatura.extract(html_content, include_comments=False)
+                if trafilatura_text:
+                    extracted_text = trafilatura_text.strip()
+                    logging.info(f"[Text Task] Trafilatura extracted fallback text, length: {len(extracted_text)}. This will be used.")
+                else:
+                    logging.warning(f"[Text Task] Trafilatura fallback extraction yielded no text.")
+            except Exception as e_traf:
+                logging.error(f"[Text Task] Trafilatura fallback error: {e_traf}")
+                if not error_message: error_message = f"Trafilatura fallback failed: {e_traf}"
+        
+        if not extracted_text and not error_message:
+             logging.warning("[Text Task] Text extraction attempts failed or yielded no text.")
+
+    except Exception as e_task_text:
+        logging.error(f"[Text Task] Unexpected error during text extraction: {e_task_text}")
+        error_message = error_message or f"Unexpected text task error: {e_task_text}"
+
+    return extracted_text, error_message
+
+# --- Task for Screenshot Capture ---
+async def _task_capture_screenshot(page, context, start_time: float) -> Tuple[Optional[bytes], Optional[str]]:
+    screenshot_bytes = None
+    error_message = None
+    cdp_session_ss = None 
+
+    try:
+        logging.info("[Screenshot Task] Starting screenshot capture process...")
+        try:
+            logging.info("[Screenshot Task] Scrolling to top and modifying fixed elements...")
+            scroll_modify_budget = get_remaining_time(start_time, budget_for_step=3)
+            if scroll_modify_budget < MIN_TIME_FOR_STEP / 2:
+                logging.warning(f"[Screenshot Task] Skipping scroll/modify due to time ({scroll_modify_budget:.2f}s)")
+            else:
+                await asyncio.wait_for(page.evaluate("window.scrollTo(0, 0)"), timeout=scroll_modify_budget / 2)
+                modified_count = await asyncio.wait_for(page.evaluate(
+                     """() => { 
+                        let m_count = 0;
+                        const allElements = document.querySelectorAll('*');
+                        allElements.forEach(el => {
+                            if (el.tagName !== 'BODY' && el.tagName !== 'HTML') {
+                                const style = window.getComputedStyle(el);
+                                if (style.position === 'fixed') {
+                                    el.style.position = 'absolute';
+                                    m_count++;
+                                }
+                            }
+                        });
+                        return m_count;
+                     }"""
+                ), timeout=scroll_modify_budget / 2)
+                logging.info(f"[Screenshot Task] {modified_count} fixed elements modified.")
+                await asyncio.sleep(0.3)
+        except Exception as e_scroll_modify:
+            logging.warning(f"[Screenshot Task] Error during scroll/modify: {e_scroll_modify}")
+
+        try:
+            logging.info("[Screenshot Task] Attempting CDP full page screenshot...")
+            metrics_budget = get_remaining_time(start_time, budget_for_step=1)
+            cdp_capture_budget = get_remaining_time(start_time, budget_for_step=10)
+
+            if metrics_budget < MIN_TIME_FOR_STEP / 2 or cdp_capture_budget < MIN_TIME_FOR_STEP:
+                 raise PlaywrightTimeoutError("Not enough time for CDP screenshot steps.")
+
+            metrics = await asyncio.wait_for(page.evaluate(
+                "() => ({ width: Math.max(document.body.scrollWidth, document.documentElement.scrollWidth), height: Math.max(document.body.scrollHeight, document.documentElement.scrollHeight) })"
+            ), timeout=metrics_budget)
+            full_width = metrics['width']
+            full_height = metrics['height']
+            if not full_width or not full_height: raise ValueError("Invalid page dimensions for CDP.")
+
+            logging.info(f"[Screenshot Task] Allocating {cdp_capture_budget:.2f}s for CDP capture.")
+            cdp_session_ss = await asyncio.wait_for(context.new_cdp_session(page), timeout=1.0)
+            cdp_result = await asyncio.wait_for(cdp_session_ss.send(
+                "Page.captureScreenshot",
+                {"format": "jpeg", "quality": 80, "captureBeyondViewport": True, "clip": {"x": 0, "y": 0, "width": full_width, "height": full_height, "scale": 1}, "fromSurface": True }
+            ), timeout=cdp_capture_budget)
+            screenshot_bytes = base64.b64decode(cdp_result['data'])
+            logging.info(f"[Screenshot Task] Captured full page screenshot via CDP (JPEG, {len(screenshot_bytes)} bytes).")
+
+        except Exception as e_ss_cdp:
+            logging.warning(f"[Screenshot Task] CDP screenshot failed: {e_ss_cdp}. Falling back.")
+            if not error_message or "closed" in str(e_ss_cdp).lower(): error_message = f"CDP Screenshot fail: {e_ss_cdp}"
+            screenshot_bytes = None
+
+        finally:
+            if cdp_session_ss:
+                try: 
+                    await asyncio.wait_for(cdp_session_ss.detach(), timeout=1.0)
+                    logging.info("[Screenshot Task] CDP session detached.")
+                except Exception as e_cdp_detach: logging.warning(f"[Screenshot Task] CDP detach error: {e_cdp_detach}")
+
+        if not screenshot_bytes:
+            try:
+                pw_full_budget = get_remaining_time(start_time, budget_for_step=8)
+                if pw_full_budget < MIN_TIME_FOR_STEP:
+                     logging.warning("[Screenshot Task] Skipping PW full page fallback due to time.")
+                else:
+                    logging.info("[Screenshot Task] Attempting Playwright full_page fallback...")
+                    screenshot_bytes = await asyncio.wait_for(page.screenshot(type='jpeg', quality=80, full_page=True), timeout=pw_full_budget)
+                    logging.info("[Screenshot Task] Captured Playwright full_page fallback.")
+            except Exception as e_ss_pw_full:
+                logging.warning(f"[Screenshot Task] PW full_page fallback failed: {e_ss_pw_full}. Trying viewport.")
+                if not error_message: error_message = f"PW full fallback fail: {e_ss_pw_full}"
+
+        if not screenshot_bytes:
+            try:
+                pw_vp_budget = get_remaining_time(start_time, budget_for_step=3)
+                if pw_vp_budget < MIN_TIME_FOR_STEP / 2:
+                     logging.warning("[Screenshot Task] Skipping PW viewport fallback due to time.")
+                else:
+                    logging.info("[Screenshot Task] Attempting Playwright viewport fallback...")
+                    screenshot_bytes = await asyncio.wait_for(page.screenshot(type='jpeg', quality=80), timeout=pw_vp_budget)
+                    logging.info("[Screenshot Task] Captured viewport fallback.")
+            except Exception as e_ss_pw_viewport:
+                 logging.warning(f"[Screenshot Task] Viewport fallback failed: {e_ss_pw_viewport}")
+                 if not error_message: error_message = f"Viewport fallback fail: {e_ss_pw_viewport}"
+
+        if not screenshot_bytes:
+            logging.error("[Screenshot Task] All screenshot attempts failed.")
+            error_message = error_message or "Screenshot capture failed despite all attempts."
+
+    except Exception as e_task_screenshot:
+        logging.error(f"[Screenshot Task] Unexpected error during screenshot capture: {e_task_screenshot}")
+        error_message = error_message or f"Unexpected screenshot task error: {e_task_screenshot}"
+        screenshot_bytes = None
+    
+    return screenshot_bytes, error_message
+
+
+async def scrape_and_screenshot_playwright(url: str, should_screenshot: bool = True) -> dict:
+    operation_start_time = time.time()
+    logging.info(f"SCRAPE START for {url}") # Log start of function execution
+
+    screenshot_bytes = None
+    extracted_text = None
+    error_message = None
+    timed_out_early = False
+    temp_data_dir_obj = None
+    playwright_instance = None
+    browser = None
+    context = None
+    page = None
+
+    try:
+        temp_data_dir_obj = tempfile.TemporaryDirectory()
+        temp_data_dir = temp_data_dir_obj.name
+        logging.info(f"Using temporary data directory for Playwright context: {temp_data_dir} for URL: {url}")
+
+        if get_remaining_time(operation_start_time) <= MIN_TIME_FOR_STEP:
+            timed_out_early = True
+            raise PlaywrightTimeoutError("Not enough time to initialize browser.")
+
+        logging.info("Attempting to start Playwright...")
+        playwright_instance = await async_playwright().start()
+        logging.info("Playwright started successfully.")
+
+        logging.info("Attempting to launch Chromium browser...")
+        browser = await playwright_instance.chromium.launch(
+            headless=True,
+            args=[
+                "--no-sandbox",
+                "--disable-setuid-sandbox",
+                "--ignore-certificate-errors",
+                "--disable-dev-shm-usage",
+                "--disable-gpu",
+            ]
+        )
+        logging.info("Chromium browser launched successfully.")
+
+        logging.info("Attempting to create browser context...")
+        context = await browser.new_context(
+            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
+            viewport={"width": 1280, "height": 720}, 
+            locale="en-US",
+        )
+        logging.info("Browser context created successfully.")
+        
+        logging.info("Attempting to create new page...")
+        page = await context.new_page()
+        logging.info("New page created successfully.")
+
+        await page.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
+
+        # Page Load
+        page_load_budget_s = get_remaining_time(operation_start_time, budget_for_step=10)
+        if page_load_budget_s <= MIN_TIME_FOR_STEP:
+            timed_out_early = True
+            raise PlaywrightTimeoutError("Not enough time for page load.")
+
+        logging.info(f"Setting page load timeout to: {page_load_budget_s:.2f}s (wait_until 'domcontentloaded')")
+        try:
+            await page.goto(url, timeout=page_load_budget_s * 1000, wait_until="domcontentloaded")
+            logging.info(f"Navigation to {url} completed (domcontentloaded).")
+        except PlaywrightTimeoutError:
+            logging.warning(f"Page load timeout for {url} (domcontentloaded strategy) within the allocated budget.")
+            timed_out_early = True
+            try: html_content_on_timeout = await page.content()
+            except: html_content_on_timeout = None
+            if should_screenshot:
+                try: 
+                    screenshot_bytes = await page.screenshot(type='jpeg', quality=80)
+                    logging.info("Captured viewport screenshot after page load timeout.")
+                except Exception as e_ss_on_timeout: logging.error(f"Failed viewport screenshot after load timeout: {e_ss_on_timeout}")
+            extracted_text = trafilatura.extract(html_content_on_timeout) if html_content_on_timeout else None
+            error_message = f"Page load timed out ({page_load_budget_s:.2f}s)"
+            raise PlaywrightTimeoutError(error_message)
+        except Exception as e_get:
+            logging.error(f"Error during page.goto({url}): {e_get}")
+            error_message = f"Error navigating to URL: {e_get}"
+            raise
+
+        # Scrolling (Sequential while loop)
+        scrolled = False
+        if not timed_out_early and get_remaining_time(operation_start_time) > MIN_TIME_FOR_STEP * 5: 
+            logging.info("Attempting to scroll page (while loop)...")
+            try:
+                last_height = await page.evaluate("document.body.scrollHeight")
+                scroll_pause_time = 1.5 
+                scroll_attempts = 0
+                max_scroll_attempts = 7
+                scroll_loop_start_time = time.time()
+                scroll_loop_budget_s = get_remaining_time(scroll_loop_start_time, budget_for_step=max_scroll_attempts * (scroll_pause_time + 0.5))
+                
+                while scroll_attempts < max_scroll_attempts:
+                    if time.time() - scroll_loop_start_time > scroll_loop_budget_s:
+                         logging.warning("Scrolling loop timed out.")
+                         timed_out_early = True; break
+                    if time.time() - operation_start_time > GLOBAL_TIMEOUT_SECONDS - BUFFER_FOR_CLEANUP_AND_PROCESSING:
+                         logging.warning("Global timeout reached during scrolling.")
+                         timed_out_early = True; break
+                    
+                    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+                    await asyncio.sleep(scroll_pause_time)
+                    new_height = await page.evaluate("document.body.scrollHeight")
+                    if new_height == last_height:
+                        scrolled = True; break
+                    last_height = new_height
+                    scroll_attempts += 1
+                
+                if scrolled: logging.info(f"Scrolling finished after {scroll_attempts} attempts.")
+                elif not timed_out_early: logging.warning(f"Scrolling completed max {max_scroll_attempts} attempts without full scroll.")
+
+            except Exception as e_scroll:
+                 logging.warning(f"Error during scrolling: {e_scroll}")
+
+        # Simplified Image Wait (Sequential)
+        if not timed_out_early and scrolled:
+            logging.info("Attempting simplified dynamic image wait (post-scroll)...")
+            await asyncio.sleep(1.0)
+
+        # Cookie Handling (Sequential)
+        cookie_banner_handled = False
+        if not timed_out_early: 
+            logging.info(f"Attempting cookie dismissal (super-fast attempts)...")
+            cookie_locator_options = [
+                {"role": "button", "name": re.compile(r"(Reject All|Tümünü Reddet|Decline All)", re.IGNORECASE)},
+                {"role": "link", "name": re.compile(r"(Reject All|Tümünü Reddet|Decline All)", re.IGNORECASE)},
+                {"role": "button", "name": re.compile(r"(Reject|Decline|Only necessary|Essential)", re.IGNORECASE)},
+                {"role": "link", "name": re.compile(r"(Reject|Decline|Only necessary|Essential)", re.IGNORECASE)},
+                {"locator_str": "button[data-testid='uc-reject-all-button']"},
+                {"locator_str": "button#uc-reject-all-button"},
+                {"role": "button", "name": re.compile(r"(Accept|Allow|Agree)( all| cookies)?", re.IGNORECASE)},
+                {"role": "link", "name": re.compile(r"(Accept|Allow|Agree)( all| cookies)?", re.IGNORECASE)},
+            ]
+            cookie_check_start_time = time.time()
+            for option in cookie_locator_options:
+                if get_remaining_time(operation_start_time) < MIN_TIME_FOR_STEP:
+                    logging.warning("Global timeout imminent during cookie checks. Halting cookie attempts.")
+                    timed_out_early = True 
+                    break
+                try:
+                    target_element = None
+                    if "locator_str" in option: target_element = page.locator(option["locator_str"]).first
+                    elif "role" in option and "name" in option: target_element = page.get_by_role(option["role"], name=option["name"]).first
+                    
+                    if target_element:
+                        logging.debug(f"Attempting ultra-fast interaction with cookie element: {option}")
+                        clicked_successfully_this_attempt = False
+                        try:
+                            await target_element.click(timeout=200)
+                            clicked_successfully_this_attempt = True
+                            logging.info(f"Playwright click likely succeeded for cookie element: {option}.")
+                        except Exception:
+                            bounding_box = None
+                            try: bounding_box = await asyncio.wait_for(target_element.bounding_box(), timeout=0.1)
+                            except: pass
+                            if bounding_box:
+                                center_x = bounding_box['x'] + bounding_box['width'] / 2
+                                center_y = bounding_box['y'] + bounding_box['height'] / 2
+                                cdp_session_cookie = None
+                                try:
+                                    cdp_session_cookie = await asyncio.wait_for(context.new_cdp_session(page), timeout=0.2)
+                                    await asyncio.wait_for(cdp_session_cookie.send("Input.dispatchMouseEvent", {"type": "mousePressed", "button": "left", "clickCount": 1, "x": center_x, "y": center_y }), timeout=0.1)
+                                    await asyncio.wait_for(cdp_session_cookie.send("Input.dispatchMouseEvent", {"type": "mouseReleased", "button": "left", "clickCount": 1, "x": center_x, "y": center_y }), timeout=0.1)
+                                    clicked_successfully_this_attempt = True
+                                    logging.info(f"Ultra-fast CDP click dispatched for cookie element: {option}.")
+                                except Exception as e_cdp_click_fast:
+                                    logging.debug(f"Ultra-fast CDP click also failed for {option}: {e_cdp_click_fast}")
+                                finally:
+                                    if cdp_session_cookie:
+                                        try:
+                                            await asyncio.wait_for(cdp_session_cookie.detach(), timeout=0.1)
+                                        except:
+                                            pass
+                            else: logging.debug(f"No bounding_box for {option} for fast CDP attempt.")
+                        
+                        if clicked_successfully_this_attempt:
+                            cookie_banner_handled = True
+                            logging.info("Cookie banner handled, breaking loop.")
+                            break
+                except Exception as e_cookie_outer: logging.debug(f"Outer error processing cookie option {option}: {e_cookie_outer}")
+            
+            logging.info(f"Finished cookie attempts. Handled: {cookie_banner_handled}, Timed Out: {timed_out_early}")
+            if not cookie_banner_handled and not timed_out_early: logging.warning("Failed to handle cookie banner.")
+
+        # --- Parallel Execution: Text Extraction and Screenshot ---
+        if not timed_out_early:
+            logging.info("Starting parallel execution of text extraction and screenshot tasks...")
+            tasks_to_run = []
+            tasks_to_run.append(_task_extract_text(page, operation_start_time))
+            if should_screenshot:
+                tasks_to_run.append(_task_capture_screenshot(page, context, operation_start_time))
+            else:
+                 tasks_to_run.append(asyncio.sleep(0, result=(None, None)))
+
+            gather_budget_s = get_remaining_time(operation_start_time)
+            if gather_budget_s <= MIN_TIME_FOR_STEP:
+                 logging.warning("Not enough time for parallel tasks. Skipping.")
+                 timed_out_early = True
+            else:
+                logging.info(f"Running asyncio.gather with remaining time: {gather_budget_s:.2f}s")
+                try:
+                    results = await asyncio.wait_for(asyncio.gather(*tasks_to_run, return_exceptions=True), timeout=gather_budget_s)
+                    
+                    # Process Text Result
+                    text_result = results[0]
+                    if isinstance(text_result, Exception):
+                        logging.error(f"Text extraction task failed with exception: {text_result}")
+                        if not error_message: error_message = f"Text task exception: {text_result}"
+                    elif text_result is not None:
+                        extracted_text, text_error = text_result
+                        if text_error and not error_message: error_message = text_error
+
+                    # Process Screenshot Result
+                    screenshot_result = results[1]
+                    if isinstance(screenshot_result, Exception):
+                        logging.error(f"Screenshot task failed with exception: {screenshot_result}")
+                        if not error_message: error_message = f"Screenshot task exception: {screenshot_result}"
+                        elif should_screenshot: error_message += f"; Screenshot task exception: {screenshot_result}"
+                    elif screenshot_result is not None:
+                        screenshot_bytes, screenshot_error = screenshot_result
+                        if screenshot_error:
+                             logging.warning(f"Screenshot task reported an error: {screenshot_error}")
+                             if not error_message and should_screenshot: error_message = screenshot_error
+                             elif should_screenshot: error_message += f"; {screenshot_error}"
+
+                except asyncio.TimeoutError:
+                    logging.warning(f"Parallel tasks timed out (gather level after {gather_budget_s:.2f}s). Partial results may be missing.")
+                    timed_out_early = True
+                    if not error_message: error_message = "Parallel execution phase timed out."
+
+        # --- End Parallel Execution ---
+
+    except PlaywrightTimeoutError as pte:
+        logging.warning(f"Playwright operation timed out for {url}: {pte}")
+        error_message = error_message or f"Processing timed out (Playwright): {pte}"
+        timed_out_early = True
+    except PlaywrightError as pe: 
+        logging.error(f"Playwright error for {url}: {pe}")
+        error_message = error_message or f"Playwright error: {pe}"
+    except Exception as e:
+        logging.error(f"General error in scrape_and_screenshot_playwright for {url}: {e}", exc_info=True)
+        error_message = error_message or f"General scraping error: {e}"
+    finally:
+        logging.info("Entering main finally block for cleanup...")
+        if page and not page.is_closed():
+            try: 
+                 await page.close()
+                 logging.info("Playwright page closed in finally block.")
+            except Exception as e_page_close: logging.warning(f"Error closing Playwright page in finally: {e_page_close}")
+        if context:
+            try: await context.close()
+            except Exception as e_ctx: logging.warning(f"Error closing Playwright context: {e_ctx}")
+        if browser:
+            try: await browser.close()
+            except Exception as e_brws: logging.warning(f"Error closing Playwright browser: {e_brws}")
+        if playwright_instance:
+            try: await playwright_instance.stop()
+            except Exception as e_pw_stop: logging.warning(f"Error stopping Playwright: {e_pw_stop}")
+        
+        if temp_data_dir_obj:
+            time.sleep(0.5)
+            try:
+                temp_data_dir_obj.cleanup()
+                logging.info(f"Successfully cleaned up temporary data directory: {temp_data_dir_obj.name}")
+            except Exception as e_cleanup:
+                logging.error(f"Error cleaning up temporary data directory {temp_data_dir_obj.name}: {e_cleanup}")
+                try: shutil.rmtree(temp_data_dir_obj.name, ignore_errors=True)
+                except Exception as e_shutil: logging.error(f"Shutil.rmtree cleanup also failed for {temp_data_dir_obj.name}: {e_shutil}")
+
+
+    # Combine results and final error message assembly
+    final_error_message = None
+    if timed_out_early:
+        timeout_info = f"Operation timed out within {GLOBAL_TIMEOUT_SECONDS}s budget."
+        if error_message and "timed out" not in error_message.lower(): final_error_message = f"{timeout_info} Last error: {error_message}"
+        elif error_message: final_error_message = error_message
+        else: final_error_message = timeout_info
+    elif error_message:
+        final_error_message = error_message
+
+    if should_screenshot and not screenshot_bytes:
+        capture_failure_message = "Screenshot requested but could not be captured."
+        if not final_error_message: final_error_message = capture_failure_message
+        elif capture_failure_message not in final_error_message: final_error_message += f"; {capture_failure_message}"
+    
+    if not extracted_text and not final_error_message:
+        final_error_message = "Text could not be extracted from the page."
+    elif not extracted_text and final_error_message and "Text could not be extracted" not in final_error_message and "timed out" not in final_error_message.lower():
+        final_error_message += "; Text also failed to extract."
+
+    final_response = {
+        "url": url,
+        "text": extracted_text or "",
+        "error": final_error_message
+    }
+    if screenshot_bytes:
+        final_response["screenshot_base64"] = base64.b64encode(screenshot_bytes).decode('utf-8')
+        
+    elapsed_total = time.time() - operation_start_time
+    logging.info(f"Total time for {url} (Playwright): {elapsed_total:.2f}s. Timed out early: {timed_out_early}. Error: {final_response.get('error')}")
+
+    return final_response
+
+@app.route(route="scrape")
+async def http_scrape_trigger(req: func.HttpRequest) -> func.HttpResponse:
+    logging.info('Python HTTP scrape trigger function processed a request.')
+    url = None
+    take_screenshot = True 
+
+    try:
+        url = req.params.get('url')
+        if url:
+            logging.info(f"Found URL in query parameters: {url}")
+            ss_param = req.params.get('take_screenshot', 'true')
+            take_screenshot = ss_param.lower() != 'false'
+        else:
+             logging.info("URL not found in query parameters.")
+    except Exception as e:
+        logging.warning(f"Error reading query parameters: {e}")
+        url = None
+
+    if not url:
+        logging.info("Attempting to read URL from JSON body.")
+        try:
+            req_body = await req.get_json()
+            if req_body:
+                url = req_body.get('url')
+                if url:
+                    logging.info(f"Found URL in JSON body: {url}")
+                    ss_param = req_body.get('take_screenshot', True)
+                    if isinstance(ss_param, str):
+                        take_screenshot = ss_param.lower() != 'false'
+                    else:
+                        take_screenshot = bool(ss_param)
+                    logging.info(f"Screenshot parameter from JSON: {take_screenshot}")
+                else:
+                    logging.info("URL key not found in JSON body.")
+            else:
+                logging.info("JSON body is empty.")
+        except ValueError:
+            logging.info("Request body is not valid JSON or missing.")
+        except Exception as e:
+            logging.warning(f"Error reading JSON body: {e}")
+            url = None
+
+    if url:
+        try:
+            if not url.startswith(('http://', 'https://')):
+                 raise ValueError("Invalid URL format. Must start with http:// or https://")
+
+            result_data = await scrape_and_screenshot_playwright(url, should_screenshot=take_screenshot)
+            
+            status_code = 200
+            if result_data.get("error") and "timed out" in result_data.get("error", "").lower():
+                 pass 
+
+            return func.HttpResponse(
+                json.dumps(result_data),
+                mimetype="application/json",
+                status_code=status_code 
+            )
+        except ValueError as ve:
+             logging.error(f"Invalid URL provided: {ve}")
+             return func.HttpResponse(
+                  json.dumps({"url": url, "error": str(ve)}),
+                  mimetype="application/json",
+                  status_code=400
+             )
+        except Exception as e:
+            logging.error(f"Error in http_scrape_trigger for {url}: {e}")
+            return func.HttpResponse(
+                 json.dumps({"url": url, "error": f"An internal error occurred in trigger: {e}"}),
+                 mimetype="application/json",
+                 status_code=500
+            )
+    else:
+        logging.warning("URL not provided in request body or query string.")
+        return func.HttpResponse(
+             json.dumps({"error": "Please pass a 'url' in the JSON request body or query string"}),
+             mimetype="application/json",
+             status_code=400
+        )
+
+
+# aiohttp server part
+async def handle_aiohttp_request(request: web.Request) -> web.Response:
+    logging.info('aiohttp /scrape endpoint hit.')
+    url = None
+    take_screenshot = True
+
+    url = request.query.get('url')
+    if url:
+        logging.info(f"Found URL in query parameters: {url}")
+        ss_param = request.query.get('take_screenshot', 'true')
+        take_screenshot = ss_param.lower() != 'false'
+    else:
+        logging.info("URL not found in query parameters. Attempting to read from JSON body.")
+        try:
+            req_body = await request.json()
+            if req_body:
+                url = req_body.get('url')
+                if url:
+                    logging.info(f"Found URL in JSON body: {url}")
+                    ss_param = req_body.get('take_screenshot', True)
+                    if isinstance(ss_param, str):
+                        take_screenshot = ss_param.lower() != 'false'
+                    else:
+                        take_screenshot = bool(ss_param)
+                    logging.info(f"Screenshot parameter from JSON: {take_screenshot}")
+                else:
+                    logging.info("URL key not found in JSON body.")
+            else:
+                logging.info("JSON body is empty or not provided.")
+        except json.JSONDecodeError:
+            logging.info("Request body is not valid JSON.")
+        except Exception as e:
+            logging.warning(f"Error reading JSON body for aiohttp request: {e}")
+            url = None
+
+    if url:
+        try:
+            if not url.startswith(('http://', 'https://')):
+                raise ValueError("Invalid URL format. Must start with http:// or https://")
+
+            result_data = await scrape_and_screenshot_playwright(url, should_screenshot=take_screenshot)
+            
+            status_code = 200
+            return web.json_response(result_data, status=status_code)
+        except ValueError as ve:
+            logging.error(f"Invalid URL provided to aiohttp server: {ve}")
+            return web.json_response({"url": url, "error": str(ve)}, status=400)
+        except Exception as e:
+            logging.error(f"Error in aiohttp_handle_request for {url}: {e}")
+            return web.json_response({"url": url, "error": f"An internal server error occurred: {e}"}, status=500)
+    else:
+        logging.warning("URL not provided in aiohttp request body or query string.")
+        return web.json_response({"error": "Please pass a 'url' in the JSON request body or query string"}, status=400)
+
+async def main_server():
+    logging.basicConfig(level=logging.INFO)
+    port_str = os.environ.get("PORT", "7777")
+    try:
+        port = int(port_str)
+    except ValueError:
+        logging.warning(f"Invalid PORT environment variable '{port_str}'. Defaulting to 7777.")
+        port = 7777
+    
+    aiohttp_app = web.Application()
+    aiohttp_app.router.add_post('/scrape', handle_aiohttp_request)
+    aiohttp_app.router.add_get('/scrape', handle_aiohttp_request)
+
+    runner = web.AppRunner(aiohttp_app)
+    await runner.setup()
+    site = web.TCPSite(runner, '0.0.0.0', port)
+    await site.start()
+    logging.info(f"aiohttp server started on http://0.0.0.0:{port}/scrape")
+    print(f"======== Running on http://0.0.0.0:{port}/scrape ========")
+    print("(Press CTRL+C to quit)")
+    try:
+        while True:
+            await asyncio.sleep(3600)
+    except KeyboardInterrupt:
+        logging.info("Server shutting down...")
+    finally:
+        await runner.cleanup()
+        logging.info("Server stopped.")
+
+if __name__ == "__main__":
+    try:
+        asyncio.run(main_server())
+    except KeyboardInterrupt:
+        logging.info("Application shut down by user.")
+    except Exception as e:
+        logging.critical(f"Application failed to start or crashed: {e}")