@capagents/forgeqe 2.3.3 → 2.3.4

package/HOW_TO_USE.md

@@ -371,6 +371,7 @@ GitHub Models, which ForgeQE used for the engine before, was [retired by GitHub]
 | `Model for agent: main, because copilot failed its live check …` | ForgeQE used your other model because Copilot did not answer; the message ends with the fix. |
 | The model is refused, or not found | The seat's plan or your organization's Copilot policies do not allow that model: pick one the sign-in listed, or ask a Copilot admin to enable it. |
 | `GitHub Copilot runs through the ForgeQE engine, which is not set up` (or `could not be set up (…)`) | Run `forgeqe setup`; the reason in brackets says what failed. |
+| Copilot is slow, or a run waits on it ("Starting the browser…", "working out the first step…") | `~/.forgeqe/logs/copilot-gateway.log` has a line for each Copilot request: when it started, Copilot's session, when each turn ended, and how long it took. It never holds prompts, replies or tokens. A turn that ends without Copilot saying it is idle (a known Copilot SDK issue) is answered 5 seconds later instead of after the 5-minute limit, and a session the run no longer needs (Stop, an error) is stopped. |
 | `CERTIFICATE_VERIFY_FAILED` while downloading Copilot's runtime | Your network inspects HTTPS. Update ForgeQE and run `forgeqe setup`, then sign in again: it uses the certificates your computer trusts (see [Company networks](#company-networks)). If it still fails, run `forgeqe doctor` and send its output to IT. |
 
 ### Your prompts, skills, and product knowledge

package/docs/user-guide.html

@@ -616,6 +616,7 @@ bun src/index.ts init --global --profile azure-gpt4o
 <tr><td><code>Model for agent: main, because copilot failed its live check …</code></td><td>ForgeQE used your other model because Copilot did not answer; the message ends with the fix.</td></tr>
 <tr><td>The model is refused, or not found</td><td>The seat's plan or your organization's Copilot policies do not allow that model: pick one the sign-in listed, or ask a Copilot admin to enable it.</td></tr>
 <tr><td><code>GitHub Copilot runs through the ForgeQE engine, which is not set up</code> (or <code>could not be set up (…)</code>)</td><td>Run <code>forgeqe setup</code>; the reason in brackets says what failed.</td></tr>
+<tr><td>Copilot is slow, or a run waits on it (&quot;Starting the browser…&quot;, &quot;working out the first step…&quot;)</td><td><code>~/.forgeqe/logs/copilot-gateway.log</code> has a line for each Copilot request: when it started, Copilot's session, when each turn ended, and how long it took. It never holds prompts, replies or tokens. A turn that ends without Copilot saying it is idle (a known Copilot SDK issue) is answered 5 seconds later instead of after the 5-minute limit, and a session the run no longer needs (Stop, an error) is stopped.</td></tr>
 <tr><td><code>CERTIFICATE_VERIFY_FAILED</code> while downloading Copilot's runtime</td><td>Your network inspects HTTPS. Update ForgeQE and run <code>forgeqe setup</code>, then sign in again: it uses the certificates your computer trusts (see <a href="../HOW_TO_USE.md#company-networks">Company networks</a>). If it still fails, run <code>forgeqe doctor</code> and send its output to IT.</td></tr>
 </tbody>
 </table></div>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@capagents/forgeqe",
-  "version": "2.3.3",
+  "version": "2.3.4",
   "description": "ForgeQE: a browser agent CLI that runs goals and test cases in local Chrome or Chromium, or BrowserStack and TestMu AI/LambdaTest when configured, and turns passing runs into tests in your repo",
   "license": "SEE LICENSE IN LICENSE",
   "repository": {

package/py/forgeqe_engine/copilot_gateway.py

@@ -24,12 +24,14 @@ from __future__ import annotations
 import asyncio
 import hmac
 import json
+import logging
 import os
 import re
 import sys
 import tempfile
 import time
 import uuid
+from logging.handlers import RotatingFileHandler
 from typing import Any, Callable, Dict, List, Optional, TextIO, Tuple
 
 from .systemcerts import use_system_certificates
@@ -40,6 +42,10 @@ KEY_ENV = "FORGEQE_COPILOT_GATEWAY_KEY"
 MAX_BODY = 32 * 1024 * 1024
 # A screenshot step can take minutes on a slow model; past this the request fails rather than hangs.
 REQUEST_TIMEOUT_S = 300.0
+# After a turn ends with its reply (or a call of the caller's tools), how long Copilot gets to say it is idle. Copilot
+# sometimes never does (github/copilot-sdk#2853), which held a request, and every agent waiting on it, for the full
+# REQUEST_TIMEOUT_S.
+TURN_END_GRACE_S = 5.0
 # A tool name as OpenAI and the Copilot SDK take it.
 TOOL_NAME = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
 MAX_TOOLS = 256
@@ -53,6 +59,34 @@ class GatewayError(Exception):
         self.message = message
 
 
+class CallerGone(Exception):
+    """The caller closed the connection (the user pressed Stop): its Copilot session is stopped, nothing is answered."""
+
+
+# What each request did and how long it took, for when Copilot is slow or stuck: never prompts, replies or tokens.
+_log = logging.getLogger("forgeqe.copilot_gateway")
+_log.propagate = False
+
+
+def gateway_log(rid: str, text: str) -> None:
+    _log.info("%s %s", rid, text)
+
+
+def open_gateway_log() -> Optional[str]:
+    """~/.forgeqe/logs/copilot-gateway.log (FORGEQE_HOME), kept to three files of 1 MB."""
+    home = os.environ.get("FORGEQE_HOME") or os.path.join(os.path.expanduser("~"), ".forgeqe")
+    path = os.path.join(home, "logs", "copilot-gateway.log")
+    try:
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        handler = RotatingFileHandler(path, maxBytes=1_000_000, backupCount=2, encoding="utf-8")
+    except OSError:
+        return None
+    handler.setFormatter(logging.Formatter("%(asctime)s %(message)s"))
+    _log.addHandler(handler)
+    _log.setLevel(logging.INFO)
+    return path
+
+
 # ---------------------------------------------------------------------------------------------------------------------
 # OpenAI chat completions → one Copilot session
 
@@ -337,11 +371,37 @@ class Copilot:
             })
         return out
 
-    async def complete(self, req: Dict[str, Any], on_text: Optional[Callable[[str], None]] = None) -> Dict[str, Any]:
+    async def complete(self, req: Dict[str, Any], on_text: Optional[Callable[[str], None]] = None, gone: Optional["asyncio.Future[Any]"] = None) -> Dict[str, Any]:
         """The answer to one request. With on_text, the reply's text goes to it as Copilot writes it (not for a JSON
-        reply, whose code fences are removed once it is whole)."""
+        reply, whose code fences are removed once it is whole). `gone` completes when the caller hangs up."""
+        rid = uuid.uuid4().hex[:8]
+        started = time.monotonic()
+        gateway_log(rid, f"start model={req['model']} tools={len(req['tools'])} schema={req['schema'] is not None} stream={bool(on_text)}")
+        try:
+            out = await self._complete(rid, req, on_text, gone)
+        except CallerGone:
+            gateway_log(rid, f"caller gone after {time.monotonic() - started:.1f}s: its Copilot session was stopped")
+            raise
+        except GatewayError as e:
+            gateway_log(rid, f"error {e.status} after {time.monotonic() - started:.1f}s: {e.message[:300]}")
+            raise
+        calls = out["choices"][0]["message"].get("tool_calls") or []
+        gateway_log(rid, f"done after {time.monotonic() - started:.1f}s" + (f" calls={','.join(c['function']['name'] for c in calls)}" if calls else ""))
+        return out
+
+    async def _complete(self, rid: str, req: Dict[str, Any], on_text: Optional[Callable[[str], None]], gone: Optional["asyncio.Future[Any]"]) -> Dict[str, Any]:
         from copilot.generated.rpc import PermissionDecisionReject
-        from copilot.session_events import AssistantMessageData, AssistantMessageDeltaData, AssistantUsageData, SessionErrorData
+        from copilot.session_events import (
+            AssistantMessageData,
+            AssistantMessageDeltaData,
+            AssistantTurnEndData,
+            AssistantTurnStartData,
+            AssistantUsageData,
+            SessionErrorData,
+            SessionIdleData,
+        )
+
+        loop = asyncio.get_running_loop()
 
         # The text callback while this request streams; None once it is asked for again without streaming.
         stream: Dict[str, Any] = {"to": None if req["schema"] is not None or req["json_only"] else on_text, "sent": False}
@@ -349,6 +409,27 @@ class Copilot:
         usage = {"input": 0, "output": 0, "cached": 0}
         errors: List[str] = []
         calls: List[Dict[str, Any]] = []
+        # The last root reply without tool requests, and the timer that ends the wait when a turn ends without idle.
+        turn: Dict[str, Any] = {"final": None, "timer": None, "settled": asyncio.Event()}
+
+        def on_loop(fn: Callable[[], None]) -> None:
+            """The SDK may call on_event from a thread of its own; timers and events belong to the server's loop."""
+            try:
+                if asyncio.get_running_loop() is loop:
+                    fn()
+                    return
+            except RuntimeError:
+                pass
+            loop.call_soon_threadsafe(fn)
+
+        def settle_soon() -> None:
+            if turn["timer"] is None:
+                turn["timer"] = loop.call_later(TURN_END_GRACE_S, turn["settled"].set)
+
+        def unsettle() -> None:
+            if turn["timer"] is not None:
+                turn["timer"].cancel()
+                turn["timer"] = None
 
         def on_event(event: Any) -> None:
             data = getattr(event, "data", None)
@@ -362,12 +443,25 @@ class Copilot:
             elif stream["to"] and isinstance(data, AssistantMessageDeltaData) and data.delta_content and not data.parent_tool_call_id:
                 stream["sent"] = True
                 stream["to"](data.delta_content)
+            elif isinstance(data, AssistantMessageData) and not data.parent_tool_call_id:
+                turn["final"] = None if data.tool_requests else event
+            elif isinstance(data, AssistantTurnStartData) and not data.parent_tool_call_id:
+                on_loop(unsettle)
+            elif isinstance(data, AssistantTurnEndData) and not data.parent_tool_call_id:
+                gateway_log(rid, "turn ended" + (" with a call of the caller's tools" if calls else " with a reply" if turn["final"] is not None else ""))
+                if calls or turn["final"] is not None:
+                    on_loop(settle_soon)
+            elif isinstance(data, SessionIdleData):
+                gateway_log(rid, "idle")
 
         tried: set = set()
         reply: Any = None
         for _ in range(MAX_TOOL_RETRIES + 2):
             errors.clear()
             calls.clear()
+            on_loop(unsettle)
+            turn["final"] = None
+            turn["settled"] = asyncio.Event()
             names = {t["name"] for t in req["tools"]}
             tools = relay_tools(req["tools"], calls, replaces=self.replaces & names)
             options: Dict[str, Any] = dict(
@@ -387,15 +481,27 @@ class Copilot:
             )
             failure: Optional[str] = None
             session = None
+            # A session that did not finish cleanly is stopped before it is detached: detaching alone leaves Copilot
+            # working on it in the background.
+            clean = False
             try:
                 session = await self.client.create_session(**options)
-                reply = await session.send_and_wait(req["prompt"], attachments=req["attachments"] or None, response_schema=req["schema"], timeout=REQUEST_TIMEOUT_S)
+                gateway_log(rid, f"session {getattr(session, 'session_id', '?')}")
+                reply, clean = await self._wait(rid, session, req, turn, gone)
             except TimeoutError:
                 raise GatewayError(504, f"GitHub Copilot did not answer within {int(REQUEST_TIMEOUT_S)} seconds.")
+            except CallerGone:
+                raise
             except Exception as e:  # Copilot names a tool it will not take; anything else is reported below.
                 failure = str(e)
             finally:
+                on_loop(unsettle)
                 if session is not None:
+                    if not clean:
+                        try:
+                            await session.abort()
+                        except Exception:
+                            pass
                     try:
                         await session.disconnect()
                     except Exception:
@@ -431,6 +537,33 @@ class Copilot:
         return completion(req["model"], content, usage)
 
 
+    async def _wait(self, rid: str, session: Any, req: Dict[str, Any], turn: Dict[str, Any], gone: Optional["asyncio.Future[Any]"]) -> Tuple[Any, bool]:
+        """
+        The reply, and whether Copilot finished cleanly. Copilot normally says it is idle; when a turn has ended with a
+        reply or a call of the caller's tools and it does not, the reply is taken TURN_END_GRACE_S later. A caller that
+        hangs up ends the wait at once.
+        """
+        waiter = asyncio.ensure_future(
+            session.send_and_wait(req["prompt"], attachments=req["attachments"] or None, response_schema=req["schema"], timeout=REQUEST_TIMEOUT_S)
+        )
+        waiter.add_done_callback(lambda t: t.cancelled() or t.exception())
+        settled = asyncio.ensure_future(turn["settled"].wait())
+        watch = {waiter, settled} | ({gone} if gone is not None else set())
+        try:
+            done, _ = await asyncio.wait(watch, timeout=REQUEST_TIMEOUT_S + 5, return_when=asyncio.FIRST_COMPLETED)
+        finally:
+            settled.cancel()
+        if waiter in done:
+            return waiter.result(), True
+        waiter.cancel()
+        if gone is not None and gone in done:
+            raise CallerGone()
+        if settled in done:
+            gateway_log(rid, f"the turn ended but Copilot did not say it was idle within {TURN_END_GRACE_S:.0f}s: answered anyway")
+            return turn["final"], False
+        raise TimeoutError()
+
+
 # Copilot's own tools that the agent runtime's tools are named like (bash, glob, grep, edit, …). A caller tool with one
 # of these names says it replaces Copilot's: the caller's tool is the one offered, and it runs nothing here.
 BUILTIN_TOOLS = frozenset({"bash", "powershell", "glob", "grep", "view", "edit", "create", "task", "skill", "web_fetch", "think", "update_todo", "report_intent"})
@@ -566,6 +699,13 @@ class Stream:
         self._event({"error": {"message": message, "type": "forgeqe_copilot_gateway"}})
 
 
+async def caller_gone(reader: asyncio.StreamReader) -> None:
+    """Completes when the caller closes the connection. Its callers (the agent runtime, the engine's HTTP client) never
+    half-close, so the end of what it sends is the caller gone."""
+    while await reader.read(4096):
+        pass
+
+
 async def handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter, copilot: Any, key: str) -> None:
     try:
         try:
@@ -608,16 +748,24 @@ async def handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter, cop
             writer.write(_error(400, "The request body is not JSON."))
             return
         req = translate(body)
-        if req["stream"]:
-            stream = Stream(writer, req["model"])
-            try:
-                await stream.finish(await copilot.complete(req, stream.text))
-            except GatewayError as e:
-                await stream.fail(e.status, e.message)
-            except Exception as e:
-                await stream.fail(502, f"GitHub Copilot: {type(e).__name__}: {e}"[:2000])
-            return
-        writer.write(_response(200, await copilot.complete(req)))
+        gone = asyncio.ensure_future(caller_gone(reader))
+        try:
+            if req["stream"]:
+                stream = Stream(writer, req["model"])
+                try:
+                    await stream.finish(await copilot.complete(req, stream.text, gone=gone))
+                except GatewayError as e:
+                    await stream.fail(e.status, e.message)
+                except CallerGone:
+                    raise
+                except Exception as e:
+                    await stream.fail(502, f"GitHub Copilot: {type(e).__name__}: {e}"[:2000])
+                return
+            writer.write(_response(200, await copilot.complete(req, gone=gone)))
+        finally:
+            gone.cancel()
+    except CallerGone:
+        return
     except GatewayError as e:
         writer.write(_error(e.status, e.message))
     except Exception as e:  # The SDK's own errors: report them rather than drop the connection.
@@ -640,6 +788,7 @@ async def serve(out: TextIO, copilot: Any = None) -> int:
         if token.startswith("ghp_"):
             out.write(json.dumps({"error": "GitHub Copilot does not take classic tokens (ghp_): use a fine-grained token with the Copilot Requests permission."}) + "\n")
             return 1
+        open_gateway_log()
         try:
             copilot = Copilot(token or None)
             await copilot.start()
@@ -654,6 +803,7 @@ async def serve(out: TextIO, copilot: Any = None) -> int:
             return 1
     server = await asyncio.start_server(lambda r, w: handle(r, w, copilot, key), host="127.0.0.1", port=0, limit=64 * 1024)
     port = server.sockets[0].getsockname()[1]
+    gateway_log("gateway", f"listening on 127.0.0.1:{port}")
     out.write(json.dumps({"port": port, "login": getattr(copilot, "login", "")}) + "\n")
     out.flush()
     loop = asyncio.get_running_loop()