@researai/deepscientist 1.5.16 → 1.5.17

package/src/deepscientist/codex_cli_compat.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import ipaddress
 import json
 import re
 import shutil
@@ -7,14 +8,22 @@ import subprocess
 import tomllib
 from functools import lru_cache
 from pathlib import Path
+from urllib.parse import urlparse
 
 from .shared import ensure_dir, read_text, write_text
 
 _MIN_XHIGH_SUPPORTED_VERSION = (0, 63, 0)
+_CHAT_WIRE_COMPAT_VERSION = (0, 57, 0)
 _CODEX_VERSION_PATTERN = re.compile(r"codex-cli\s+(\d+)\.(\d+)\.(\d+)", re.IGNORECASE)
 _CODEX_HOME_SYNCED_FILES = ("config.toml", "auth.json")
 _CODEX_HOME_SYNCED_DIRS = ("skills", "agents", "prompts")
 _CODEX_HOME_QUEST_OVERLAY_DIRS = ("skills", "prompts")
+_ROOT_TABLE_SECTION_PATTERN = re.compile(r"^\s*\[")
+_ROOT_MODEL_ASSIGNMENT_PATTERN = re.compile(r"^\s*(model_provider|model)\s*=")
+_COMPAT_BEGIN_MARKER = "# BEGIN DEEPSCIENTIST PROFILE COMPAT"
+_COMPAT_END_MARKER = "# END DEEPSCIENTIST PROFILE COMPAT"
+_MISSING_ENV_PATTERN = re.compile(r"Missing environment variable:\s*[`'\"]?([^`'\"\s]+)", re.IGNORECASE)
+_LOCAL_PROVIDER_HOST_ALIASES = {"localhost", "host.docker.internal"}
 
 
 def parse_codex_cli_version(text: str) -> tuple[int, int, int] | None:
@@ -48,6 +57,50 @@ def format_codex_cli_version(version: tuple[int, int, int] | None) -> str:
     return ".".join(str(part) for part in version)
 
 
+def chat_wire_compatible_codex_version() -> tuple[int, int, int]:
+    return _CHAT_WIRE_COMPAT_VERSION
+
+
+def _split_root_table_lines(config_text: str) -> tuple[list[str], list[str]]:
+    lines = str(config_text or "").splitlines()
+    for index, line in enumerate(lines):
+        if _ROOT_TABLE_SECTION_PATTERN.match(line):
+            return lines[:index], lines[index:]
+    return lines, []
+
+
+def _strip_root_model_assignments(lines: list[str]) -> list[str]:
+    filtered: list[str] = []
+    skipping_compat_block = False
+    for line in lines:
+        stripped = line.strip()
+        if stripped == _COMPAT_BEGIN_MARKER:
+            skipping_compat_block = True
+            continue
+        if skipping_compat_block:
+            if stripped == _COMPAT_END_MARKER:
+                skipping_compat_block = False
+            continue
+        if _ROOT_MODEL_ASSIGNMENT_PATTERN.match(line):
+            continue
+        filtered.append(line)
+    while filtered and not filtered[0].strip():
+        filtered.pop(0)
+    while filtered and not filtered[-1].strip():
+        filtered.pop()
+    return filtered
+
+
+def _join_field_names(fields: list[str]) -> str:
+    if not fields:
+        return ""
+    if len(fields) == 1:
+        return fields[0]
+    if len(fields) == 2:
+        return f"{fields[0]} and {fields[1]}"
+    return ", ".join(fields[:-1]) + f", and {fields[-1]}"
+
+
 def normalize_codex_reasoning_effort(
     reasoning_effort: str | None,
     *,
@@ -93,33 +146,54 @@ def adapt_profile_only_provider_config(
     if not isinstance(profile_payload, dict):
         return config_text, None
 
-    prefix_lines: list[str] = []
-    injected_fields: list[str] = []
-    if "model_provider" not in parsed:
-        model_provider = str(profile_payload.get("model_provider") or "").strip()
-        if model_provider:
-            prefix_lines.append(f"model_provider = {json.dumps(model_provider, ensure_ascii=False)}")
-            injected_fields.append("model_provider")
-    if "model" not in parsed:
-        model = str(profile_payload.get("model") or "").strip()
-        if model:
-            prefix_lines.append(f"model = {json.dumps(model, ensure_ascii=False)}")
-            injected_fields.append("model")
-
-    if not prefix_lines:
+    profile_model_provider = str(profile_payload.get("model_provider") or "").strip()
+    profile_model = str(profile_payload.get("model") or "").strip()
+    top_level_model_provider = str(parsed.get("model_provider") or "").strip()
+    top_level_model = str(parsed.get("model") or "").strip()
+
+    root_lines: list[str] = []
+    changed_fields: list[str] = []
+    conflicted_fields: list[str] = []
+    if profile_model_provider and top_level_model_provider != profile_model_provider:
+        root_lines.append(f"model_provider = {json.dumps(profile_model_provider, ensure_ascii=False)}")
+        changed_fields.append("model_provider")
+        if top_level_model_provider:
+            conflicted_fields.append("model_provider")
+    elif profile_model_provider:
+        root_lines.append(f"model_provider = {json.dumps(profile_model_provider, ensure_ascii=False)}")
+    if profile_model and top_level_model != profile_model:
+        root_lines.append(f"model = {json.dumps(profile_model, ensure_ascii=False)}")
+        changed_fields.append("model")
+        if top_level_model:
+            conflicted_fields.append("model")
+    elif profile_model:
+        root_lines.append(f"model = {json.dumps(profile_model, ensure_ascii=False)}")
+
+    if not changed_fields:
         return config_text, None
 
-    adapted = (
-        "# BEGIN DEEPSCIENTIST PROFILE COMPAT\n"
-        + "\n".join(prefix_lines)
-        + "\n# END DEEPSCIENTIST PROFILE COMPAT\n\n"
-        + config_text.lstrip()
-    )
+    root_prefix, body_lines = _split_root_table_lines(config_text)
+    cleaned_root = _strip_root_model_assignments(root_prefix)
+    adapted_lines: list[str] = [
+        _COMPAT_BEGIN_MARKER,
+        *root_lines,
+        _COMPAT_END_MARKER,
+    ]
+    if cleaned_root:
+        adapted_lines.append("")
+        adapted_lines.extend(cleaned_root)
+    if body_lines:
+        adapted_lines.append("")
+        adapted_lines.extend(body_lines)
+    adapted = "\n".join(adapted_lines).rstrip() + "\n"
+    field_text = _join_field_names(changed_fields)
     return (
         adapted,
         (
-            f"DeepScientist promoted `{normalized_profile}` profile "
-            f"{', '.join(injected_fields)} to the top level for Codex compatibility."
+            f"DeepScientist overrode conflicting top-level {field_text} with values from profile "
+            f"`{normalized_profile}` for Codex compatibility."
+            if conflicted_fields
+            else f"DeepScientist promoted `{normalized_profile}` profile {field_text} to the top level for Codex compatibility."
         ),
     )
 
@@ -235,61 +309,47 @@ def materialize_codex_runtime_home(
     return warning
 
 
-def provider_profile_metadata(
+def _empty_provider_metadata() -> dict[str, str | bool | None]:
+    return {
+        "provider": None,
+        "model": None,
+        "env_key": None,
+        "base_url": None,
+        "wire_api": None,
+        "requires_openai_auth": None,
+    }
+
+
+def active_provider_metadata(
     config_text: str,
     *,
-    profile: str,
+    profile: str | None = None,
 ) -> dict[str, str | bool | None]:
     normalized_profile = str(profile or "").strip()
-    if not normalized_profile or not str(config_text or "").strip():
-        return {
-            "provider": None,
-            "model": None,
-            "env_key": None,
-            "base_url": None,
-            "wire_api": None,
-            "requires_openai_auth": None,
-        }
+    if not str(config_text or "").strip():
+        return _empty_provider_metadata()
     try:
         parsed = tomllib.loads(config_text)
     except tomllib.TOMLDecodeError:
-        return {
-            "provider": None,
-            "model": None,
-            "env_key": None,
-            "base_url": None,
-            "wire_api": None,
-            "requires_openai_auth": None,
-        }
-
-    profiles = parsed.get("profiles")
-    if not isinstance(profiles, dict):
-        return {
-            "provider": None,
-            "model": None,
-            "env_key": None,
-            "base_url": None,
-            "wire_api": None,
-            "requires_openai_auth": None,
-        }
-    profile_payload = profiles.get(normalized_profile)
-    if not isinstance(profile_payload, dict):
-        return {
-            "provider": None,
-            "model": None,
-            "env_key": None,
-            "base_url": None,
-            "wire_api": None,
-            "requires_openai_auth": None,
-        }
+        return _empty_provider_metadata()
+
+    profile_payload: dict | None = None
+    if normalized_profile:
+        profiles = parsed.get("profiles")
+        if not isinstance(profiles, dict):
+            return _empty_provider_metadata()
+        candidate_profile = profiles.get(normalized_profile)
+        if not isinstance(candidate_profile, dict):
+            return _empty_provider_metadata()
+        profile_payload = candidate_profile
 
     model_provider = str(
-        profile_payload.get("model_provider")
+        (profile_payload or {}).get("model_provider")
         or parsed.get("model_provider")
         or ""
     ).strip() or None
     model = str(
-        profile_payload.get("model")
+        (profile_payload or {}).get("model")
         or parsed.get("model")
         or ""
     ).strip() or None
@@ -331,6 +391,17 @@ def provider_profile_metadata(
     }
 
 
+def provider_profile_metadata(
+    config_text: str,
+    *,
+    profile: str,
+) -> dict[str, str | bool | None]:
+    normalized_profile = str(profile or "").strip()
+    if not normalized_profile:
+        return _empty_provider_metadata()
+    return active_provider_metadata(config_text, profile=normalized_profile)
+
+
 def provider_profile_metadata_from_home(
     config_home: str | Path,
     *,
@@ -338,12 +409,54 @@ def provider_profile_metadata_from_home(
 ) -> dict[str, str | bool | None]:
     config_path = Path(config_home).expanduser() / "config.toml"
     if not config_path.exists():
-        return {
-            "provider": None,
-            "model": None,
-            "env_key": None,
-            "base_url": None,
-            "wire_api": None,
-            "requires_openai_auth": None,
-        }
+        return _empty_provider_metadata()
     return provider_profile_metadata(config_path.read_text(encoding="utf-8"), profile=profile)
+
+
+def provider_base_url_looks_local(base_url: str | None) -> bool:
+    normalized = str(base_url or "").strip()
+    if not normalized:
+        return False
+    parsed = urlparse(normalized)
+    hostname = str(parsed.hostname or "").strip().lower()
+    if not hostname:
+        return False
+    if hostname in _LOCAL_PROVIDER_HOST_ALIASES or hostname.endswith(".local"):
+        return True
+    try:
+        ip = ipaddress.ip_address(hostname)
+    except ValueError:
+        return False
+    return ip.is_loopback or ip.is_private or ip.is_link_local or ip.is_unspecified
+
+
+def missing_provider_env_key(
+    metadata: dict[str, str | bool | None],
+    env: dict[str, str] | None,
+) -> str | None:
+    env_key = str((metadata or {}).get("env_key") or "").strip()
+    if not env_key:
+        return None
+    env_value = str((env or {}).get(env_key) or "").strip()
+    if env_value:
+        return None
+    return env_key
+
+
+def missing_provider_env_key_from_text(*texts: str) -> str | None:
+    for text in texts:
+        match = _MISSING_ENV_PATTERN.search(str(text or ""))
+        if match:
+            return str(match.group(1) or "").strip() or None
+    return None
+
+
+def active_provider_metadata_from_home(
+    config_home: str | Path,
+    *,
+    profile: str | None = None,
+) -> dict[str, str | bool | None]:
+    config_path = Path(config_home).expanduser() / "config.toml"
+    if not config_path.exists():
+        return _empty_provider_metadata()
+    return active_provider_metadata(config_path.read_text(encoding="utf-8"), profile=profile)

package/src/deepscientist/config/service.py

@@ -10,10 +10,16 @@ from urllib.error import URLError
 from urllib.request import Request
 
 from ..codex_cli_compat import (
+    active_provider_metadata_from_home,
     adapt_profile_only_provider_config,
+    chat_wire_compatible_codex_version,
+    codex_cli_version,
+    format_codex_cli_version,
     materialize_codex_runtime_home,
+    missing_provider_env_key,
+    missing_provider_env_key_from_text,
     normalize_codex_reasoning_effort,
-    provider_profile_metadata_from_home,
+    provider_base_url_looks_local,
 )
 from ..connector.connector_profiles import PROFILEABLE_CONNECTOR_NAMES, list_connector_profiles, normalize_connector_config
 from ..connector_runtime import build_discovered_target, infer_connector_transport
@@ -1267,32 +1273,142 @@ Use **Test** when the file exposes runtime dependencies.
                 ]
             )
         else:
-            guidance.append("Run `codex --login` (or `codex`) once and finish authentication before starting DeepScientist.")
+            guidance.append("Run `codex login` (or just `codex`) once and finish authentication before starting DeepScientist.")
         guidance.append(
             "If you use a custom Codex path, either set `runners.codex.binary` or launch with `ds --codex /absolute/path/to/codex`."
         )
         return guidance
 
+    @staticmethod
+    def _provider_profile_probe_hints(metadata: dict[str, object]) -> list[str]:
+        base_url = str(metadata.get("base_url") or "").strip().lower()
+        model = str(metadata.get("model") or "").strip().lower()
+        provider = str(metadata.get("provider") or "").strip().lower()
+        if "dashscope.aliyuncs.com" not in base_url and "bailian" not in provider and "qwen" not in model:
+            return []
+        if "coding.dashscope.aliyuncs.com" not in base_url:
+            return [
+                "Alibaba Bailian's generic DashScope / Qwen platform API is not supported by the Codex-backed DeepScientist path.",
+                "If you want to use Qwen here, switch the profile to the Bailian Coding Plan endpoint: `https://coding.dashscope.aliyuncs.com/v1`.",
+            ]
+        return [
+            "For Qwen on Alibaba Bailian, only the Coding Plan endpoint is supported here; do not switch back to the generic Bailian / DashScope Qwen API.",
+        ]
+
+    @staticmethod
+    def _local_provider_probe_hints(metadata: dict[str, object]) -> list[str]:
+        base_url = str(metadata.get("base_url") or "").strip()
+        wire_api = str(metadata.get("wire_api") or "").strip().lower()
+        requires_openai_auth = metadata.get("requires_openai_auth")
+        if not base_url:
+            return []
+        is_local_provider = provider_base_url_looks_local(base_url)
+        if requires_openai_auth is not False and not is_local_provider:
+            return []
+        hints = [
+            f"Verify the local provider directly: `curl {base_url}/models`.",
+            f"Then verify the Responses API explicitly: `curl {base_url}/responses ...`.",
+            "Latest Codex CLI requires `wire_api = \"responses\"`; chat-only provider configs are no longer accepted.",
+            "If `/v1/chat/completions` works but `/v1/responses` fails, that backend is not currently compatible with the latest Codex runner.",
+            "If the backend is chat-only and you still want to test it through Codex, try `@openai/codex@0.57.0` with top-level `model_provider` / `model` plus `wire_api = \"chat\"`.",
+            "For local model backends, vLLM is the safest path. Ollama only works when its `/v1/responses` endpoint works; chat-only SGLang deployments will fail with the latest Codex.",
+        ]
+        if requires_openai_auth is not False:
+            hints.insert(
+                0,
+                "For local or self-hosted providers, add `requires_openai_auth = false` so DeepScientist can remove conflicting `OPENAI_*` auth variables.",
+            )
+        if not wire_api:
+            hints.insert(0, "Your current provider config does not declare `wire_api`; set `wire_api = \"responses\"` first.")
+        elif wire_api != "responses":
+            hints.insert(0, f"Your current provider config uses `wire_api = \"{wire_api}\"`; switch it to `wire_api = \"responses\"` first.")
+        return hints
+
+    @staticmethod
+    def _missing_provider_env_guidance(
+        *,
+        profile: str,
+        env_key: str,
+        metadata: dict[str, object],
+    ) -> list[str]:
+        guidance = [
+            f"Set `runners.codex.env.{env_key}` in `~/DeepScientist/config/runners.yaml`, or export `{env_key}` before launching `ds`.",
+        ]
+        if provider_base_url_looks_local(str(metadata.get("base_url") or "").strip()):
+            guidance.append(
+                f"If `{env_key}` is only a placeholder for a local OpenAI-compatible backend, any non-empty value such as `1234` is usually enough."
+            )
+            if metadata.get("requires_openai_auth") is not False:
+                guidance.append(
+                    "Also add `requires_openai_auth = false` to that local provider profile so DeepScientist can remove conflicting `OPENAI_*` auth variables."
+                )
+        guidance.append(
+            f"Before retrying DeepScientist, run a real request such as `codex exec --profile {profile} --json --cd /tmp --skip-git-repo-check -` and verify it returns `HELLO`."
+        )
+        return guidance
+
+    @staticmethod
+    def _chat_wire_probe_version_block(
+        metadata: dict[str, object],
+        *,
+        resolved_binary: str,
+    ) -> tuple[tuple[int, int, int] | None, dict[str, object] | None]:
+        wire_api = str(metadata.get("wire_api") or "").strip().lower()
+        if wire_api != "chat":
+            return None, None
+        detected_version = codex_cli_version(str(resolved_binary or ""))
+        required_version = chat_wire_compatible_codex_version()
+        if detected_version == required_version:
+            return detected_version, None
+        required_text = format_codex_cli_version(required_version)
+        detected_text = format_codex_cli_version(detected_version)
+        errors = [
+            "This provider uses `wire_api = \"chat\"`, but DeepScientist only probes chat-mode providers with `codex-cli 0.57.0`.",
+        ]
+        if detected_text:
+            errors.append(f"Detected Codex CLI version: `{detected_text}`.")
+        else:
+            errors.append("DeepScientist could not determine the active Codex CLI version from the configured binary.")
+        guidance = [
+            "Install `npm install -g @openai/codex@0.57.0`, or point DeepScientist at a dedicated `0.57.0` binary with `ds --codex /absolute/path/to/codex`.",
+            "If you want to stay on a newer Codex CLI, switch the provider/backend to `wire_api = \"responses\"` instead.",
+            "For chat-mode fallback configs, keep the compatible top-level `model_provider` / `model` entries in `~/.codex/config.toml`.",
+        ]
+        return (
+            detected_version,
+            {
+                "summary": f"Codex startup probe blocked by chat-mode provider compatibility. Required Codex CLI: `{required_text}`.",
+                "errors": errors,
+                "guidance": guidance,
+            },
+        )
+
     def _codex_probe_failure_guidance(self, config: dict) -> tuple[list[str], list[str]]:
         profile = self._codex_profile_name(config)
+        config_dir = str(config.get("config_dir") or "~/.codex").strip()
+        metadata = active_provider_metadata_from_home(config_dir, profile=profile or None) if config_dir else {}
         if profile:
+            provider_hints = self._provider_profile_probe_hints(metadata)
+            local_hints = self._local_provider_probe_hints(metadata)
             return (
                 [
                     f"Codex profile `{profile}` did not complete the startup hello probe successfully.",
                 ],
                 [
-                    f"Run `codex --profile {profile}` in a terminal and confirm that profile can start normally.",
+                    f"Run `codex exec --profile {profile} --json --cd /tmp --skip-git-repo-check -` in a terminal and confirm that a real `HELLO` request succeeds.",
                     "If the profile uses a custom provider, make sure its API key, Base URL, and model configuration are available to Codex.",
                     "If the provider expects the model from the Codex profile itself, set `model: inherit` in `~/DeepScientist/config/runners.yaml`.",
+                    *provider_hints,
+                    *local_hints,
                     "Then run `ds doctor` and start DeepScientist again.",
                 ],
             )
         return (
             [
-                "Run `codex --login` (or `codex`) once and complete login before starting DeepScientist.",
+                "Run `codex login` (or just `codex`) once and complete login before starting DeepScientist.",
             ],
             [
-                "Run `codex --login` (or `codex`) in a terminal and complete login or first-run setup.",
+                "Run `codex login` (or just `codex`) in a terminal and complete login or first-run setup.",
                 "If `codex` is missing, install it explicitly with `npm install -g @openai/codex`.",
                 "If the configured model is not available to your Codex account, update `~/DeepScientist/config/runners.yaml` and try again.",
                 "Then run `ds doctor` and start DeepScientist again.",
@@ -1430,10 +1546,20 @@ Use **Test** when the file exposes runtime dependencies.
             env["CODEX_HOME"] = prepared_home
             if profile_config_warning:
                 compatibility_warnings.append(profile_config_warning)
-        metadata = provider_profile_metadata_from_home(env.get("CODEX_HOME") or config_dir, profile=profile)
+        metadata = active_provider_metadata_from_home(env.get("CODEX_HOME") or config_dir, profile=profile or None)
         if metadata.get("requires_openai_auth") is False:
             env.pop("OPENAI_API_KEY", None)
             env.pop("OPENAI_BASE_URL", None)
+        configured_provider_env_key = missing_provider_env_key(metadata, env)
+        details["provider_env_key"] = str(metadata.get("env_key") or "").strip() or None
+        details["provider_env_missing"] = bool(configured_provider_env_key)
+        details["provider_wire_api"] = str(metadata.get("wire_api") or "").strip() or None
+        detected_codex_version, chat_wire_block = self._chat_wire_probe_version_block(
+            metadata,
+            resolved_binary=resolved_binary,
+        )
+        if detected_codex_version is not None:
+            details["codex_cli_version"] = format_codex_cli_version(detected_codex_version) or None
         prompt = "Reply with exactly HELLO."
         if reasoning_effort_warning:
             compatibility_warnings.append(reasoning_effort_warning)
@@ -1442,6 +1568,15 @@ Use **Test** when the file exposes runtime dependencies.
                 f"Codex profile `{profile}` is provider-backed. DeepScientist is probing it with `model: inherit`."
             )
         base_warnings: list[str] = list(compatibility_warnings)
+        if chat_wire_block is not None:
+            return {
+                "ok": False,
+                "summary": str(chat_wire_block["summary"]),
+                "warnings": base_warnings,
+                "errors": list(chat_wire_block["errors"]),
+                "details": details,
+                "guidance": list(chat_wire_block["guidance"]),
+            }
 
         def run_probe_once(model_for_command: str) -> tuple[list[str], subprocess.CompletedProcess[str] | None, subprocess.TimeoutExpired | None]:
             command = self._build_codex_probe_command(
@@ -1568,7 +1703,20 @@ Use **Test** when the file exposes runtime dependencies.
             if details.get("model_fallback_attempted") and not details.get("model_fallback_used"):
                 warnings.append("DeepScientist also tried the current Codex default model, but that fallback probe did not succeed.")
             errors.extend(self._codex_probe_failure_guidance(config)[0])
+        missing_env_key = missing_provider_env_key_from_text(stdout_text, stderr_text) or configured_provider_env_key
         failure_guidance = self._codex_probe_failure_guidance(config)[1]
+        if not ok and missing_env_key and profile:
+            errors.append(
+                f"Codex profile `{profile}` requires environment variable `{missing_env_key}`, but DeepScientist did not receive it."
+            )
+            failure_guidance = [
+                *self._missing_provider_env_guidance(
+                    profile=profile,
+                    env_key=missing_env_key,
+                    metadata=metadata,
+                ),
+                *failure_guidance,
+            ]
         return {
             "ok": ok,
             "summary": "Codex startup probe completed." if ok else "Codex startup probe failed.",