@capagents/forgeqe 0.0.0-stage → 1.0.0

package/py/forgeqe_engine/settings.py

@@ -0,0 +1,93 @@
+"""Engine settings read from the environment: library tuning, headless mode, and the default sign-in."""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any, Dict, Optional, Tuple
+
+logger = logging.getLogger("forgeqe_engine.service")
+
+
+def _truthy_env(name: str, default: bool = False) -> bool:
+    raw = os.getenv(name)
+    if raw is None:
+        return default
+    return str(raw).strip().lower() in {"1", "true", "yes", "on"}
+
+
+def read_default_browser_login_credentials() -> Optional[Tuple[str, str]]:
+    """(username, password) from BROWSER_USE_DEFAULT_LOGIN_USERNAME / _PASSWORD when both are set."""
+    if _truthy_env("BROWSER_USE_SKIP_DEFAULT_LOGIN_CREDENTIALS"):
+        return None
+    user = (os.environ.get("BROWSER_USE_DEFAULT_LOGIN_USERNAME") or "").strip()
+    password = (os.environ.get("BROWSER_USE_DEFAULT_LOGIN_PASSWORD") or "").strip()
+    if not user and not password:
+        return None
+    if not user or not password:
+        logger.warning("default login is incomplete (need both username and password); skipping")
+        return None
+    return user, password
+
+
+def build_default_browser_login_system_extension() -> Optional[str]:
+    creds = read_default_browser_login_credentials()
+    if not creds:
+        return None
+    user, password = creds
+    return f"""## Default application sign-in (use when the scenario requires authentication)
+When the user goal involves **logging in**, **sign in**, **authentication**, or you reach a **login / password** form and the goal did **not** specify different credentials, fill the form using:
+- **Email or username:** `{user}`
+- **Password:** `{password}`
+
+Prefer fields labeled Email, Username, or Login together with Password. Submit the form and wait for a successful post-login state before continuing. If the page already shows a signed-in session, do not re-enter credentials unless the flow requires re-auth.""".strip()
+
+
+def _browser_use_agent_tuning_kwargs() -> Dict[str, Any]:
+    """
+    Library 0.12+ defaults (planning, thinking, judge, vision) add large per-step latency
+    on heavy sites (e.g. booking.com). Fast mode is on unless BROWSER_USE_FULL_AGENT_MODE=1.
+    """
+    if _truthy_env("BROWSER_USE_FULL_AGENT_MODE"):
+        return {}
+    if not _truthy_env("BROWSER_USE_FAST_MODE", default=True):
+        return {}
+    tuning: Dict[str, Any] = {
+        "enable_planning": _truthy_env("BROWSER_USE_ENABLE_PLANNING", default=False),
+        "use_thinking": _truthy_env("BROWSER_USE_USE_THINKING", default=False),
+        # One model call after done that checks the trace and screenshots against the goal; its verdict decides success.
+        # Off by default (browser.judge): it failed runs whose answer came from one screenshot rather than a step per item.
+        "use_judge": _truthy_env("BROWSER_USE_USE_JUDGE", default=False),
+        "flash_mode": _truthy_env("BROWSER_USE_FLASH_MODE", default=True),
+        "loop_detection_enabled": _truthy_env("BROWSER_USE_LOOP_DETECTION", default=True),
+        # Later steps of a long run were resending every earlier step, so exploration got slower
+        # as it went. Keep the first step and the most recent ones. Summarising the rest is another
+        # model call on the step path, which costs more time than it saves.
+        "max_history_items": 8,
+        "message_compaction": False,
+    }
+    loop_window_raw = (os.getenv("BROWSER_USE_LOOP_DETECTION_WINDOW") or "").strip()
+    if loop_window_raw.isdigit():
+        tuning["loop_detection_window"] = max(5, int(loop_window_raw))
+    history_raw = (os.getenv("BROWSER_USE_MAX_HISTORY_ITEMS") or "").strip()
+    if history_raw.isdigit():
+        tuning["max_history_items"] = max(6, int(history_raw))
+    if _truthy_env("BROWSER_USE_LOW_VISION_DETAIL", default=True):
+        tuning["vision_detail_level"] = "low"
+    max_actions = (os.getenv("BROWSER_USE_MAX_ACTIONS_PER_STEP") or "").strip()
+    if max_actions.isdigit():
+        tuning["max_actions_per_step"] = max(1, int(max_actions))
+    llm_timeout_raw = (os.getenv("BROWSER_USE_LLM_TIMEOUT") or "").strip()
+    if llm_timeout_raw.isdigit():
+        tuning["llm_timeout"] = max(30, int(llm_timeout_raw))
+    max_failures_raw = (os.getenv("BROWSER_USE_MAX_FAILURES") or "").strip()
+    if max_failures_raw.isdigit():
+        tuning["max_failures"] = max(1, int(max_failures_raw))
+    return tuning
+
+
+def resolve_browser_headless(headed: Optional[bool] = None) -> bool:
+    """headed=True → visible browser. When omitted, BROWSER_USE_HEADLESS decides (default visible)."""
+    if headed is not None:
+        return not bool(headed)
+    return _truthy_env("BROWSER_USE_HEADLESS", default=False)

package/py/forgeqe_engine/skills.py

@@ -0,0 +1,88 @@
+"""Skills for the browser agent: choosing which ones fit a goal, and the prompt text for the chosen ones."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any, Dict, List, Tuple
+
+from .prompts import text as prompt_text
+
+logger = logging.getLogger("forgeqe_engine")
+
+MAX_CHOSEN = 3
+# The choice runs before the agent, outside the run's own timeout: a model that hangs here costs at most this long.
+SELECT_TIMEOUT_SECONDS = 60.0
+
+SELECT_SYSTEM = (
+    "You pick playbooks for a browser automation agent. Each playbook explains how to work with a particular kind of "
+    "website or widget. Given the agent's goal and the list of playbooks, return the names of the playbooks the goal "
+    "clearly needs (at most 3), or an empty list when none fits. Never pick a playbook only because it could be loosely related."
+)
+
+
+def _clean(catalog: List[Dict[str, Any]]) -> List[Dict[str, str]]:
+    out: List[Dict[str, str]] = []
+    for s in catalog or []:
+        if not isinstance(s, dict):
+            continue
+        name = str(s.get("name") or "").strip()
+        body = str(s.get("body") or "").strip()
+        if name and body:
+            out.append({"name": name, "description": str(s.get("description") or "").strip(), "body": body})
+    return out
+
+
+async def choose_skills(llm: Any, goal: str, catalog: List[Dict[str, Any]]) -> Tuple[List[str], Dict[str, int]]:
+    """Names from ``catalog`` that fit ``goal``, in catalogue order, and the tokens the choice took. Never raises."""
+    skills = _clean(catalog)
+    usage = {"input": 0, "output": 0}
+    if not skills:
+        return [], usage
+    try:
+        from browser_use.llm.messages import SystemMessage, UserMessage
+        from pydantic import BaseModel
+
+        class Choice(BaseModel):
+            skills: List[str]
+
+        listing = "\n".join(f"- {s['name']}: {s['description']}" for s in skills)
+        res = await asyncio.wait_for(
+            llm.ainvoke(
+                [SystemMessage(content=prompt_text("browser-skill-select", SELECT_SYSTEM)), UserMessage(content=f"Goal:\n{goal}\n\nPlaybooks:\n{listing}")],
+                output_format=Choice,
+            ),
+            timeout=SELECT_TIMEOUT_SECONDS,
+        )
+        u = getattr(res, "usage", None)
+        if u is not None:
+            usage = {"input": int(getattr(u, "prompt_tokens", 0) or 0), "output": int(getattr(u, "completion_tokens", 0) or 0)}
+        picked = {str(n).strip() for n in (getattr(res.completion, "skills", None) or [])}
+    except Exception as ex:
+        logger.warning("skill selection failed, running without skills: %s", ex)
+        return [], usage
+    return [s["name"] for s in skills if s["name"] in picked][:MAX_CHOSEN], usage
+
+
+def skills_block(chosen: List[str], catalog: List[Dict[str, Any]], max_chars: int) -> Tuple[str, List[str], List[str]]:
+    """Prompt text for the chosen skills within ``max_chars``: (text, names included, names left out for size)."""
+    by_name = {s["name"]: s for s in _clean(catalog)}
+    parts: List[str] = []
+    used: List[str] = []
+    dropped: List[str] = []
+    size = 0
+    for name in chosen:
+        s = by_name.get(name)
+        if s is None:
+            continue
+        part = f"## Skill {name}\n\n{s['body']}"
+        if size + len(part) > max_chars:
+            dropped.append(name)
+            continue
+        parts.append(part)
+        used.append(name)
+        size += len(part) + 2
+    if not parts:
+        return "", used, dropped
+    head = "Skills for this run (follow them where they apply; the goal and the rules above come first):"
+    return head + "\n\n" + "\n\n".join(parts), used, dropped

package/py/forgeqe_engine/summary.py

@@ -0,0 +1,239 @@
+"""Summaries and result markdown built from the agent's history."""
+
+from __future__ import annotations
+
+import base64
+import json
+import re
+from collections import Counter
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+from .settings import _truthy_env
+
+
+def _compact_url_journey(urls: List[str]) -> Optional[str]:
+    if not urls:
+        return None
+    seen: List[str] = []
+    for u in urls:
+        if u and u not in seen:
+            seen.append(u)
+    if not seen:
+        return None
+    if len(seen) == 1:
+        return f"**Page:** `{seen[0]}`"
+    a, b = seen[0], seen[-1]
+    if a == b:
+        return f"**Page:** `{a}`"
+    return f"**Journey:** `{a}` → … → `{b}` (`{len(seen)}` pages)"
+
+
+def _fallback_summary_from_history(history: Any) -> str:
+    """When final_result() is empty, derive text from the trace."""
+    try:
+        for h in reversed(history.history or []):
+            if not h.model_output:
+                continue
+            for act in h.model_output.action:
+                try:
+                    d = act.model_dump(exclude_none=True, mode="json")
+                except Exception:
+                    continue
+                done = d.get("done") if isinstance(d, dict) else None
+                if isinstance(done, dict):
+                    t = done.get("text")
+                    if t and str(t).strip():
+                        return str(t).strip()
+    except Exception:
+        pass
+    try:
+        exts = history.extracted_content()
+    except Exception:
+        exts = []
+    for block in reversed(exts or []):
+        txt = str(block).strip()
+        if len(txt) < 28:
+            continue
+        if txt.lower().startswith(("clicked ", "typed ", "waited ", "🔗")):
+            continue
+        return txt[:8000]
+    try:
+        if history.history:
+            mo = history.history[-1].model_output
+            if mo:
+                joined = " ".join(str(p) for p in (mo.memory, mo.next_goal, mo.evaluation_previous_goal) if p).strip()
+                if joined:
+                    return joined[:4000]
+    except Exception:
+        pass
+    return ""
+
+
+def _error_summary_from_history(history: Any) -> str:
+    """When the model or transport fails repeatedly, surface a clear message instead of an empty result."""
+    try:
+        errs = [str(e).strip() for e in history.errors() if e and str(e).strip()]
+        if not errs:
+            return ""
+        counts = Counter(errs)
+        distinct = list(counts.keys())
+        n = len(errs)
+        if len(distinct) == 1:
+            msg = distinct[0]
+            if n > 1:
+                return (
+                    f"Run stopped after {n} identical errors: {msg}. "
+                    "This usually means the LLM provider could not be reached or rejected the request — "
+                    "check the profile's credentials (its API key; for auth: entra, that the identity has the "
+                    "Cognitive Services OpenAI User role on the resource), endpoint, network, and quotas."
+                )
+            return f"Run stopped: {msg}. Verify LLM configuration and connectivity if this is a connection or auth error."
+        return (
+            f"Run stopped after {n} errors ({len(distinct)} distinct). Last: {distinct[-1]}. "
+            "Review LLM provider settings and the error trace."
+        )
+    except Exception:
+        return ""
+
+
+def _primary_summary(history: Any) -> str:
+    try:
+        fr = history.final_result()
+        if fr and str(fr).strip():
+            return str(fr).strip()
+    except Exception:
+        pass
+    return _fallback_summary_from_history(history) or _error_summary_from_history(history)
+
+
+def _failure_trace_markdown(history: Any) -> str:
+    lines: List[str] = ["### Error trace", ""]
+    try:
+        errs = [str(e).strip() for e in history.errors() if e and str(e).strip()]
+        if errs:
+            for msg, cnt in Counter(errs).most_common(12):
+                lines.append(f"- `{msg}`" + (f" (×{cnt})" if cnt > 1 else ""))
+            lines.append("")
+    except Exception:
+        pass
+    try:
+        steps = history.agent_steps()
+        if steps:
+            lines.append("### Recent agent steps (abbrev.)")
+            lines.append("")
+            for s in steps[-8:]:
+                chunk = str(s).strip()
+                if len(chunk) > 1200:
+                    chunk = chunk[:1100] + "…"
+                lines.extend(["```", chunk, "```", ""])
+    except Exception:
+        pass
+    return "\n".join(lines).strip()
+
+
+def _failure_screenshot_asset(history: Any) -> Optional[Tuple[str, bytes]]:
+    """(extension, bytes) for a real PNG/JPEG of the last state, or None."""
+    try:
+        fn = getattr(history, "screenshots", None)
+        if not callable(fn):
+            return None
+        shots = [s for s in (fn() or []) if s]
+        if not shots:
+            return None
+        last = shots[-1]
+        raw_bytes: Optional[bytes] = None
+        b64_guess: Optional[str] = None
+        if isinstance(last, str):
+            if last.startswith("data:image") and "," in last:
+                b64_guess = last.split(",", 1)[-1].strip()
+            elif len(last) > 80 and re.match(r"^[A-Za-z0-9+/=\s]+$", last[: min(200, len(last))]):
+                b64_guess = re.sub(r"\s+", "", last)
+            elif Path(last).is_file():
+                raw_bytes = Path(last).read_bytes()
+        elif isinstance(last, (bytes, bytearray)):
+            raw_bytes = bytes(last)
+        if raw_bytes is None and b64_guess:
+            b64_guess += "=" * ((-len(b64_guess)) % 4)
+            try:
+                raw_bytes = base64.b64decode(b64_guess, validate=True)
+            except Exception:
+                raw_bytes = None
+        if not raw_bytes or len(raw_bytes) < 64:
+            return None
+        if raw_bytes.startswith(b"\x89PNG\r\n\x1a\n"):
+            return ("png", raw_bytes)
+        if raw_bytes.startswith(b"\xff\xd8\xff"):
+            return ("jpg", raw_bytes)
+    except Exception:
+        pass
+    return None
+
+
+def build_browser_result_markdown(history: Any, checks: Optional[List[Dict[str, Any]]] = None) -> str:
+    """Human-readable markdown: the agent's own summary, the checks it recorded, compact URL line."""
+    verbose_md = _truthy_env("BROWSER_USE_VERBOSE_MARKDOWN")
+    summary_str = _primary_summary(history)
+
+    lines: List[str] = ["## Result", "", summary_str or "_Task finished with no summary text._", ""]
+
+    try:
+        ext = history.extracted_content()
+    except Exception:
+        ext = []
+
+    if checks:
+        lines.extend(["### Checks", ""])
+        lines += [f"- {c.get('expected') or c.get('text')}: \"{c.get('text')}\"" for c in checks]
+        lines.append("")
+
+    try:
+        journey = _compact_url_journey([u for u in (history.urls() or []) if u])
+    except Exception:
+        journey = None
+    if journey:
+        lines.extend([journey, ""])
+
+    try:
+        n_hist = len(history.history or [])
+    except Exception:
+        n_hist = 0
+    if n_hist:
+        lines.extend(["### Run metadata", "", f"- **LLM steps in trace:** {n_hist}", ""])
+
+    if verbose_md:
+        if ext:
+            lines.extend(["### All extractions (verbose)", ""])
+            for i, block in enumerate(ext, 1):
+                txt = str(block).strip() if block else ""
+                if txt:
+                    lines.extend([f"_{i}._ {txt}", ""])
+        try:
+            ah = history.action_history()
+        except Exception:
+            ah = []
+        if ah:
+            lines.extend(["### Actions per step (verbose)", "", "| Step | Action |", "|------|--------|"])
+            for si, step_actions in enumerate(ah, 1):
+                for a in step_actions:
+                    if not isinstance(a, dict):
+                        continue
+                    slim = {k: v for k, v in a.items() if k != "interacted_element" and v is not None}
+                    cell = json.dumps(slim, ensure_ascii=False, default=str)[:450].replace("|", "\\|")
+                    lines.append(f"| {si} | `{cell}` |")
+            lines.append("")
+
+    out = "\n".join(lines).strip()
+    return out or summary_str or "_No structured details were captured._"
+
+
+def judge_from_history(history: Any) -> Optional[Dict[str, Any]]:
+    """The library judge's verdict on the finished run, or None when the judge did not run."""
+    try:
+        j = history.judgement()
+    except Exception:
+        return None
+    if not isinstance(j, dict) or not isinstance(j.get("verdict"), bool):
+        return None
+    keep = ("verdict", "reasoning", "failure_reason", "impossible_task", "reached_captcha")
+    return {k: j[k] for k in keep if j.get(k) not in (None, "")}

package/py/forgeqe_engine/urls.py

@@ -0,0 +1,81 @@
+"""Start URLs and the allowed-hosts list."""
+
+from __future__ import annotations
+
+import os
+import re
+from typing import List, Optional
+from urllib.parse import urlparse
+
+
+def _parse_allowed_domains(explicit: Optional[List[str]] = None) -> Optional[List[str]]:
+    parts = [p.strip().lower() for p in (explicit or []) if p and p.strip()]
+    if parts:
+        return parts
+    raw = (os.getenv("BROWSER_USE_ALLOWED_HOSTS") or "").strip()
+    if not raw:
+        return None
+    parts = [p.strip().lower() for p in raw.split(",") if p.strip()]
+    return parts or None
+
+
+def _extract_urls(text: str) -> List[str]:
+    return re.findall(r"https?://[^\s<>\"']+", text or "")
+
+
+def _hostname_allowed(hostname: str, allowed: Optional[List[str]]) -> bool:
+    if not allowed:
+        return True
+    h = (hostname or "").lower().rstrip(".")
+    for pattern in allowed:
+        p = pattern.strip().lower()
+        if not p:
+            continue
+        if p.startswith("*."):
+            suffix = p[1:]
+            if h.endswith(suffix) or h == p[2:]:
+                return True
+        elif h == p or h.endswith("." + p):
+            return True
+    return False
+
+
+def _allowed_hosts_violation_for_url(url: str, allowed: Optional[List[str]]) -> Optional[str]:
+    """Return an error message if this URL's host is outside the allow-list (when set)."""
+    if not allowed:
+        return None
+    try:
+        host = urlparse(url).hostname or ""
+    except Exception:
+        return f"Invalid URL in task: {url}"
+    if not host:
+        return None
+    if not _hostname_allowed(host, allowed):
+        return f"Host not allowed by allowed_hosts: {host}"
+    return None
+
+
+def _validate_goal_urls_against_allowlist(goal: str, allowed: Optional[List[str]]) -> Optional[str]:
+    for url in _extract_urls(goal):
+        err = _allowed_hosts_violation_for_url(url, allowed)
+        if err:
+            return err
+    return None
+
+
+def normalize_http_base_url(raw: Optional[str]) -> Optional[str]:
+    """Trim and ensure an http(s) scheme. None if empty, unparsable, or missing a host."""
+    if raw is None:
+        return None
+    u = str(raw).strip()
+    if not u:
+        return None
+    if not re.match(r"^[a-zA-Z][a-zA-Z0-9+.-]*://", u):
+        u = "https://" + u.lstrip("/")
+    try:
+        parsed = urlparse(u)
+    except Exception:
+        return None
+    if not parsed.scheme or not parsed.netloc:
+        return None
+    return u