@clawpump/claw-agent 0.1.5 → 0.1.6

package/agent/agent/transports/anthropic.py

@@ -84,39 +84,73 @@ class AnthropicTransport(ProviderTransport):
         to OpenAI finish_reason, and collects reasoning_details in provider_data.
         """
         import json
-        from agent.anthropic_adapter import _to_plain_data
+        from agent.anthropic_adapter import _to_plain_data, _sanitize_replay_block
         from agent.transports.types import ToolCall
 
         strip_tool_prefix = kwargs.get("strip_tool_prefix", False)
-        _MCP_PREFIX = "mcp_"
+        _MCP_PREFIX = "mcp__"
 
         text_parts = []
         reasoning_parts = []
         reasoning_details = []
         tool_calls = []
+        # Verbatim, order-preserving copy of every content block in the turn.
+        # Anthropic signs each thinking block against the turn content that
+        # PRECEDES it at its position; when a turn interleaves thinking and
+        # tool_use (adaptive/interleaved thinking, Claude 4.6+), the parallel
+        # reasoning_details + tool_calls lists below lose that cross-type
+        # ordering. Replaying the latest assistant message in the wrong order
+        # invalidates the signatures -> HTTP 400 "thinking ... blocks in the
+        # latest assistant message cannot be modified". Preserve the exact
+        # block sequence here so the adapter can replay it unchanged. See
+        # tests/agent/test_anthropic_thinking_block_order.py.
+        ordered_blocks = []
 
         for block in response.content:
+            block_dict = _to_plain_data(block)
+            clean_block = None
+            if isinstance(block_dict, dict):
+                # Sanitize at capture so output-only SDK fields (parsed_output,
+                # caller, citations=None, …) never persist to state.db and leak
+                # back as request input on replay → HTTP 400 "Extra inputs are
+                # not permitted". Defence-in-depth with the replay-side sanitize.
+                clean_block = _sanitize_replay_block(block_dict)
+                if clean_block is not None:
+                    ordered_blocks.append(clean_block)
             if block.type == "text":
                 text_parts.append(block.text)
-            elif block.type == "thinking":
-                reasoning_parts.append(block.thinking)
-                block_dict = _to_plain_data(block)
-                if isinstance(block_dict, dict):
+            elif block.type in ("thinking", "redacted_thinking"):
+                if block.type == "thinking":
+                    reasoning_parts.append(block.thinking)
+                # Use the sanitized block (clean_block) for reasoning_details too,
+                # since _extract_preserved_thinking_blocks replays these on the
+                # non-ordered path. Falls back to raw only if sanitize dropped it.
+                if isinstance(clean_block, dict):
+                    reasoning_details.append(clean_block)
+                elif isinstance(block_dict, dict):
                     reasoning_details.append(block_dict)
             elif block.type == "tool_use":
                 name = block.name
                 if strip_tool_prefix and name.startswith(_MCP_PREFIX):
-                    stripped = name[len(_MCP_PREFIX):]
-                    # Only strip the mcp_ prefix for OAuth-injected tools
-                    # (where Hermes adds the prefix when sending to Anthropic
-                    # and must remove it on the way back).  Native MCP server
-                    # tools (from mcp_servers: in config.yaml) are registered
-                    # in the tool registry under their FULL mcp_<server>_<tool>
-                    # name and must NOT be stripped.  GH-25255.
+                    # On the OAuth wire every tool carries a double-underscore
+                    # ``mcp__`` prefix (added in build_anthropic_kwargs to avoid
+                    # Anthropic's single-underscore third-party classifier).
+                    # Reverse it back to the name the registry/dispatcher knows.
+                    # Two original forms map onto the same ``mcp__`` wire name:
+                    #   ``mcp__read_file``       <- bare native tool ``read_file``
+                    #   ``mcp__linear_get_issue`` <- MCP server tool
+                    #                                ``mcp_linear_get_issue``
+                    # Resolve by registry lookup, preferring whichever original
+                    # is actually registered; never rewrite a name the LLM used
+                    # that already resolves natively. GH-25255.
                     from tools.registry import registry as _tool_registry
-                    if (_tool_registry.get_entry(stripped)
-                            and not _tool_registry.get_entry(name)):
-                        name = stripped
+                    if not _tool_registry.get_entry(name):
+                        bare = name[len(_MCP_PREFIX):]            # read_file
+                        single = "mcp_" + bare                    # mcp_read_file / mcp_linear_get_issue
+                        if _tool_registry.get_entry(single):
+                            name = single
+                        elif _tool_registry.get_entry(bare):
+                            name = bare
                 tool_calls.append(
                     ToolCall(
                         id=block.id,
@@ -130,6 +164,23 @@ class AnthropicTransport(ProviderTransport):
         provider_data = {}
         if reasoning_details:
             provider_data["reasoning_details"] = reasoning_details
+        # Only worth carrying the ordered-blocks channel when the turn
+        # actually interleaves signed thinking with tool_use — that's the
+        # only shape the parallel lists reconstruct incorrectly. A turn that
+        # is purely text, or thinking-then-tools with a single leading
+        # thinking block, replays correctly without it.
+        _has_signed_thinking = any(
+            isinstance(b, dict)
+            and b.get("type") in ("thinking", "redacted_thinking")
+            and (b.get("signature") or b.get("data"))
+            for b in ordered_blocks
+        )
+        _has_tool_use = any(
+            isinstance(b, dict) and b.get("type") == "tool_use"
+            for b in ordered_blocks
+        )
+        if _has_signed_thinking and _has_tool_use:
+            provider_data["anthropic_content_blocks"] = ordered_blocks
 
         return NormalizedResponse(
             content="\n".join(text_parts) if text_parts else None,
@@ -143,10 +194,21 @@ class AnthropicTransport(ProviderTransport):
     def validate_response(self, response: Any) -> bool:
         """Check Anthropic response structure is valid.
 
-        An empty content list is legitimate when ``stop_reason == "end_turn"``
-        — the model's canonical way of signalling "nothing more to add" after
-        a tool turn that already delivered the user-facing text. Treating it
-        as invalid falsely retries a completed response.
+        An empty content list is legitimate for terminal stop reasons that
+        carry no text payload:
+
+        - ``end_turn`` — the model's canonical "nothing more to add" after a
+          tool turn that already delivered the user-facing text.
+        - ``refusal`` — the model declined to respond (Claude 4.5+). The
+          Messages API returns an empty ``content`` list with this stop
+          reason. Treating it as invalid sends a deterministic refusal into
+          the invalid-response retry loop, which reproduces the refusal on
+          every attempt and surfaces a misleading "rate limited / invalid
+          response" error instead of the refusal. ``normalize_response`` maps
+          ``refusal`` → ``content_filter`` so the agent loop's refusal handler
+          can surface it.
+
+        Treating either as invalid falsely retries a completed response.
         """
         if response is None:
             return False
@@ -154,7 +216,7 @@ class AnthropicTransport(ProviderTransport):
         if not isinstance(content_blocks, list):
             return False
         if not content_blocks:
-            return getattr(response, "stop_reason", None) == "end_turn"
+            return getattr(response, "stop_reason", None) in {"end_turn", "refusal"}
         return True
 
     def extract_cache_stats(self, response: Any) -> Optional[Dict[str, int]]:

package/agent/agent/transports/chat_completions.py

@@ -99,6 +99,22 @@ def _is_gemini_openai_compat_base_url(base_url: Any) -> bool:
     return normalized.endswith("/openai")
 
 
+def _model_consumes_thought_signature(model: Any) -> bool:
+    """True when the outgoing model is a Gemini family model that requires
+    ``extra_content`` (thought_signature) to be replayed on tool calls.
+
+    Gemini 3 thinking models attach ``extra_content`` to each tool call and
+    reject subsequent requests with HTTP 400 if it is missing. Every other
+    strict OpenAI-compatible provider (Fireworks, Mistral, ...) rejects the
+    request with 400 if ``extra_content`` *is* present. So the field must be
+    kept only when the target model is itself Gemini-family, and stripped
+    otherwise — including when a non-Gemini model inherits stale Gemini
+    ``extra_content`` from earlier in a mixed-provider session.
+    """
+    m = str(model or "").lower()
+    return "gemini" in m or "gemma" in m
+
+
 class ChatCompletionsTransport(ProviderTransport):
     """Transport for api_mode='chat_completions'.
 
@@ -119,6 +135,14 @@ class ChatCompletionsTransport(ProviderTransport):
         - Codex Responses API fields: ``codex_reasoning_items`` /
           ``codex_message_items`` on the message, ``call_id`` /
           ``response_item_id`` on ``tool_calls`` entries.
+        - ``extra_content`` on ``tool_calls`` (Gemini thought_signature) —
+          stripped unless the outgoing ``model`` is itself Gemini-family.
+          Gemini 3 thinking models attach it for replay, but strict providers
+          (Fireworks, Mistral) reject any payload containing it with
+          ``Extra inputs are not permitted, field: 'messages[N].tool_calls[M].extra_content'``.
+          It must be kept for Gemini targets (replay required) and dropped for
+          everyone else, including non-Gemini models that inherited stale
+          Gemini ``extra_content`` earlier in a mixed-provider session.
         - ``tool_name`` on tool-result messages — written by
           ``make_tool_result_message()`` for the SQLite FTS index, but not
           part of the Chat Completions schema. Strict providers (Fireworks,
@@ -137,6 +161,9 @@ class ChatCompletionsTransport(ProviderTransport):
           ``Extra inputs are not permitted, field: 'messages[N]._empty_recovery_synthetic'``,
           which then poisons every subsequent request in the session.
         """
+        strip_extra_content = not _model_consumes_thought_signature(
+            kwargs.get("model")
+        )
         needs_sanitize = False
         for msg in messages:
             if not isinstance(msg, dict):
@@ -155,7 +182,9 @@ class ChatCompletionsTransport(ProviderTransport):
             if isinstance(tool_calls, list):
                 for tc in tool_calls:
                     if isinstance(tc, dict) and (
-                        "call_id" in tc or "response_item_id" in tc
+                        "call_id" in tc
+                        or "response_item_id" in tc
+                        or (strip_extra_content and "extra_content" in tc)
                     ):
                         needs_sanitize = True
                         break
@@ -183,6 +212,8 @@ class ChatCompletionsTransport(ProviderTransport):
                     if isinstance(tc, dict):
                         tc.pop("call_id", None)
                         tc.pop("response_item_id", None)
+                        if strip_extra_content:
+                            tc.pop("extra_content", None)
         return sanitized
 
     def convert_tools(self, tools: list[dict[str, Any]]) -> list[dict[str, Any]]:
@@ -240,8 +271,10 @@ class ChatCompletionsTransport(ProviderTransport):
             anthropic_max_output: int | None
             extra_body_additions: dict | None
         """
-        # Codex sanitization: drop reasoning_items / call_id / response_item_id
-        sanitized = self.convert_messages(messages)
+        # Codex sanitization: drop reasoning_items / call_id / response_item_id.
+        # Pass model so the Gemini thought_signature (extra_content) is kept for
+        # Gemini targets and stripped for strict non-Gemini providers.
+        sanitized = self.convert_messages(messages, model=model)
 
         # ── Provider profile: single-path when present ──────────────────
         _profile = params.get("provider_profile")
@@ -498,6 +531,7 @@ class ChatCompletionsTransport(ProviderTransport):
                 supports_reasoning=params.get("supports_reasoning", False),
                 qwen_session_metadata=params.get("qwen_session_metadata"),
                 model=model,
+                base_url=params.get("base_url"),
                 ollama_num_ctx=params.get("ollama_num_ctx"),
                 session_id=params.get("session_id"),
             )
@@ -538,7 +572,28 @@ class ChatCompletionsTransport(ProviderTransport):
                     api_kwargs[k] = v
 
         if extra_body:
-            api_kwargs["extra_body"] = extra_body
+            # Native Gemini (generativelanguage.googleapis.com, non-/openai)
+            # speaks Google's REST schema, not OpenAI's. OpenAI-style extra_body
+            # keys (tags, reasoning, provider, plugins, …) are unknown fields
+            # there and Gemini rejects the whole request with a non-retryable
+            # HTTP 400 ("Invalid JSON payload received. Unknown name 'tags'").
+            # This happens when a profile that emits extra_body (e.g. the Nous
+            # profile's portal `tags`) is active but the resolved endpoint is a
+            # Gemini base_url — typical when only Google credentials are set and
+            # a fallback/aux call lands on Gemini. The native client only reads
+            # thinking_config from extra_body, so drop everything else here.
+            try:
+                from agent.gemini_native_adapter import is_native_gemini_base_url
+                _native_gemini = is_native_gemini_base_url(params.get("base_url"))
+            except Exception:
+                _native_gemini = False
+            if _native_gemini:
+                extra_body = {
+                    k: v for k, v in extra_body.items()
+                    if k in ("thinking_config", "thinkingConfig")
+                }
+            if extra_body:
+                api_kwargs["extra_body"] = extra_body
 
         return api_kwargs
 
@@ -610,8 +665,42 @@ class ChatCompletionsTransport(ProviderTransport):
         if rd:
             provider_data["reasoning_details"] = rd
 
+        # OpenAI structured-refusal field. When a model declines, the SDK
+        # populates ``message.refusal`` with the explanation and leaves
+        # ``content`` empty. OpenAI-compatible proxies that front Anthropic /
+        # Bedrock (e.g. Nous Portal) surface a Claude refusal this way — or via
+        # ``finish_reason="content_filter"`` — instead of the native
+        # ``stop_reason="refusal"``. Without capturing it the refusal looks
+        # like an empty response, so the agent loop retries a deterministic
+        # refusal three times and gives up with "no content after retries".
+        # Promote it to content + a ``content_filter`` finish reason so the
+        # loop's refusal handler surfaces it clearly and stops. ``refusal`` is
+        # ``None`` for normal responses, so this is a no-op in the common case.
+        content = msg.content
+        refusal = getattr(msg, "refusal", None)
+        if refusal is None and hasattr(msg, "model_extra"):
+            _msg_extra = getattr(msg, "model_extra", None) or {}
+            if isinstance(_msg_extra, dict):
+                refusal = _msg_extra.get("refusal")
+        if isinstance(refusal, str) and refusal.strip():
+            # Record the refusal explanation regardless — it's useful provider
+            # metadata even when the model also returned a usable payload.
+            provider_data["refusal"] = refusal
+            _has_text = isinstance(content, str) and content.strip()
+            _has_tool_calls = bool(tool_calls)
+            # Only promote to a terminal ``content_filter`` when the refusal is
+            # the *sole* payload — no visible text and no tool calls. A response
+            # that carries real content (or tool calls) alongside a refusal note
+            # is a normal, usable turn: surfacing it as a failed safety refusal
+            # would discard the model's actual work. In the empty-payload case,
+            # adopt the refusal as content so the loop has something to show.
+            if not _has_text and not _has_tool_calls:
+                content = refusal
+                if finish_reason in (None, "stop"):
+                    finish_reason = "content_filter"
+
         return NormalizedResponse(
-            content=msg.content,
+            content=content,
             tool_calls=tool_calls,
             finish_reason=finish_reason,
             reasoning=reasoning,

package/agent/agent/transports/codex.py

@@ -128,6 +128,65 @@ class ResponsesApiTransport(ProviderTransport):
         reasoning_effort = _effort_clamp.get(reasoning_effort, reasoning_effort)
 
         response_tools = _responses_tools(tools)
+
+        # xAI server-side web search.
+        #
+        # grok models on xAI's /v1/responses surface (notably
+        # grok-composer-2.5-fast on SuperGrok OAuth) have a *native*,
+        # server-executed web search.  When the model is handed a
+        # client-side function literally named ``web_search``, it routes
+        # the intent to that native engine — but because the tool is
+        # declared as a plain ``function`` rather than xAI's first-class
+        # ``{"type": "web_search"}`` built-in, the server-side search is
+        # dispatched but never reconciled: the response streams reasoning
+        # + ``web_search_call`` progress items, the searches never reach
+        # ``status="completed"`` in the assembled output, no final
+        # message is emitted, and ``_normalize_codex_response`` correctly
+        # sees reasoning-with-no-answer and reports ``incomplete``.  The
+        # turn then burns 3 continuation retries and fails with "Codex
+        # response remained incomplete after 3 continuation attempts".
+        # Verified live against grok-composer-2.5-fast (2026-06).
+        #
+        # Fix: when the agent HAS a client-side ``web_search`` function (i.e.
+        # the user enabled the web toolset), declare xAI's native
+        # ``web_search`` built-in instead so the search actually runs to
+        # completion server-side and the model streams a real answer.  The
+        # Responses API rejects two tools sharing the name ``web_search``
+        # (HTTP 400 "Duplicate tool names"), so we drop the client-side
+        # ``web_search`` function for the xAI path and let the native tool
+        # satisfy it.  All other client-side tools (read_file, terminal,
+        # web_extract, MCP tools, …) are untouched and continue to dispatch
+        # through Hermes's agent loop.
+        #
+        # Scope: we ONLY swap in the native built-in when the client
+        # ``web_search`` was actually present.  We do NOT force-enable Grok
+        # server-side search on turns where the user never had web enabled —
+        # that would silently route around Hermes's web-provider config and
+        # tool-trace/citation plumbing for every xai-oauth turn.  The swap is
+        # a 1:1 replacement of an already-requested capability, not an
+        # additive grant.
+        #
+        # NOTE: for the swapped case this routes ``web_search`` to Grok's
+        # native search engine for xAI sessions instead of Hermes's
+        # configured web provider (Tavily/etc.), and those results bypass
+        # Hermes's tool-trace / citation plumbing (they arrive baked into the
+        # model's answer rather than as a tool result the loop observes).
+        # Scoped to ``is_xai_responses`` deliberately; narrow to specific
+        # models if a future grok variant should keep the client-side
+        # function.
+        if is_xai_responses and response_tools:
+            has_client_web_search = any(
+                isinstance(t, dict) and t.get("name") == "web_search"
+                for t in response_tools
+            )
+            if has_client_web_search:
+                filtered = [
+                    t for t in response_tools
+                    if not (isinstance(t, dict) and t.get("name") == "web_search")
+                ]
+                filtered.append({"type": "web_search"})
+                response_tools = filtered
+
         # ``tools`` MUST be omitted entirely when there are no functions to
         # expose: the openai SDK's ``responses.stream()`` / ``responses.parse()``
         # eagerly call ``_make_tools(tools)`` which does ``for tool in tools``
@@ -218,8 +277,14 @@ class ResponsesApiTransport(ProviderTransport):
             kwargs.pop("timeout", None)
 
         if is_codex_backend:
-            prompt_cache_key = kwargs.get("prompt_cache_key")
-            cache_scope_id = str(prompt_cache_key or session_id or "").strip()
+            # The Codex backend rejects body-level ``extra_headers`` with
+            # HTTP 400, but the OpenAI SDK's ``extra_headers`` kwarg maps
+            # to actual HTTP request headers (not body fields).  We need
+            # these headers for cache-scope routing so prompt cache hits
+            # remain high.  Send session_id / x-client-request-id as HTTP
+            # headers while keeping ``prompt_cache_key`` in the body for
+            # standard OpenAI routing as a belt-and-braces fallback.
+            cache_scope_id = str(session_id or "").strip()
             if cache_scope_id:
                 existing_extra_headers = kwargs.get("extra_headers")
                 merged_extra_headers: Dict[str, str] = {}

package/agent/agent/transports/codex_app_server.py

@@ -378,6 +378,7 @@ def check_codex_binary(
             capture_output=True,
             text=True,
             timeout=10,
+            stdin=subprocess.DEVNULL,
         )
     except FileNotFoundError:
         return False, (

package/agent/agent/transports/codex_app_server_session.py

@@ -72,6 +72,9 @@ class TurnResult:
     error: Optional[str] = None  # Set if turn ended in a non-recoverable error
     turn_id: Optional[str] = None
     thread_id: Optional[str] = None
+    token_usage_last: Optional[dict[str, Any]] = None
+    token_usage_total: Optional[dict[str, Any]] = None
+    model_context_window: Optional[int] = None
     # Hint to the caller that the underlying codex subprocess is likely
     # wedged (turn-level timeout fired, post-tool watchdog tripped, or
     # token-refresh failure killed the child). The caller should retire
@@ -501,6 +504,7 @@ class CodexAppServerSession:
                     pending = self._client.take_notification(timeout=0)
                     if pending is None:
                         break
+                    _apply_token_usage_notification(result, pending)
                     self._track_pending_file_change(pending)
                     proj = projector.project(pending)
                     if proj.messages:
@@ -536,6 +540,8 @@ class CodexAppServerSession:
                 except Exception:  # pragma: no cover - display callback
                     logger.debug("on_event callback raised", exc_info=True)
 
+            _apply_token_usage_notification(result, note)
+
             # Track in-progress fileChange items so the approval bridge
             # can surface a real change summary when codex requests
             # approval (the approval params themselves don't carry the
@@ -802,6 +808,30 @@ class CodexAppServerSession:
         return cached
 
 
+def _apply_token_usage_notification(result: TurnResult, note: dict) -> None:
+    """Capture Codex app-server token usage updates for caller accounting.
+
+    Codex does not put token usage on turn/completed. It emits a separate
+    thread/tokenUsage/updated notification containing cumulative totals and
+    the latest turn breakdown.
+    """
+    if not isinstance(note, dict) or note.get("method") != "thread/tokenUsage/updated":
+        return
+    params = note.get("params") or {}
+    token_usage = params.get("tokenUsage") or {}
+    if not isinstance(token_usage, dict):
+        return
+    last = token_usage.get("last")
+    total = token_usage.get("total")
+    if isinstance(last, dict):
+        result.token_usage_last = dict(last)
+    if isinstance(total, dict):
+        result.token_usage_total = dict(total)
+    window = token_usage.get("modelContextWindow")
+    if isinstance(window, int) and window > 0:
+        result.model_context_window = window
+
+
 def _approval_choice_to_codex_decision(choice: str) -> str:
     """Map Hermes approval choices onto codex's CommandExecutionApprovalDecision
     / FileChangeApprovalDecision wire values.

package/agent/agent/transports/types.py

@@ -121,6 +121,18 @@ class NormalizedResponse:
         pd = self.provider_data or {}
         return pd.get("reasoning_details")
 
+    @property
+    def anthropic_content_blocks(self):
+        """Verbatim, order-preserving Anthropic content blocks for a turn.
+
+        Present only when an Anthropic turn interleaves signed thinking with
+        tool_use — the one shape the parallel reasoning_details + tool_calls
+        lists reconstruct in the wrong order, invalidating thinking-block
+        signatures on replay. See agent/transports/anthropic.py.
+        """
+        pd = self.provider_data or {}
+        return pd.get("anthropic_content_blocks")
+
     @property
     def codex_reasoning_items(self):
         pd = self.provider_data or {}