@researai/deepscientist 1.5.9 → 1.5.12

package/src/deepscientist/doctor.py

@@ -263,10 +263,7 @@ def _check_codex(config_manager: ConfigManager) -> dict[str, Any]:
             ok=False,
             summary="Codex CLI is not available to DeepScientist.",
             errors=[f"Runner binary `{binary}` could not be resolved."],
-            guidance=[
-                "Run `npm install -g @researai/deepscientist` again so the bundled Codex dependency is installed.",
-                "Then run `codex` once and complete login.",
-            ],
+            guidance=config_manager._codex_missing_binary_guidance(codex_cfg),
             details={"binary": binary},
         )
 
@@ -285,7 +282,9 @@ def _check_codex(config_manager: ConfigManager) -> dict[str, Any]:
             details={"resolved_binary": resolved_binary},
         )
     if not probe_guidance:
-        probe_guidance = ["Run `codex` manually once and complete login, then retry `ds doctor`."]
+        probe_guidance = [
+            "Run `codex --login` (or `codex`) manually once and complete login, then retry `ds doctor`.",
+        ]
     return _make_check(
         check_id="codex",
         label="Codex CLI",

package/src/deepscientist/gitops/diff.py

@@ -299,45 +299,41 @@ def _collect_branch_state(repo: Path) -> dict[str, dict[str, Any]]:
             continue
         state = branch_state[branch_name]
         state.setdefault("branch", branch_name)
-        state["updated_at"] = record.get("updated_at") or state.get("updated_at")
-        if record.get("kind") == "run":
-            state["run_id"] = record.get("run_id") or state.get("run_id")
-            state["run_kind"] = record.get("run_kind") or state.get("run_kind")
+        artifact_sort_key = _artifact_record_sort_key(record, path)
+        current_artifact_sort_key = state.get("_latest_artifact_sort_key")
+        if current_artifact_sort_key is None or artifact_sort_key > current_artifact_sort_key:
+            state["_latest_artifact_sort_key"] = artifact_sort_key
+            state["updated_at"] = record.get("updated_at") or record.get("created_at") or state.get("updated_at")
         if record.get("idea_id"):
             state["idea_id"] = record.get("idea_id")
         if record.get("parent_branch"):
             state["parent_branch"] = record.get("parent_branch")
         if record.get("worktree_root"):
             state["worktree_root"] = record.get("worktree_root")
-        latest_metric = extract_latest_metric(record)
+        resolved_run_result = _resolve_run_result_payload(repo, record)
+        latest_metric = extract_latest_metric(resolved_run_result if record.get("kind") == "run" else record)
         if latest_metric is not None:
             state["latest_metric"] = latest_metric
         if record.get("kind") == "run":
-            state["latest_result"] = {
-                "run_id": record.get("run_id"),
-                "run_kind": record.get("run_kind"),
-                "status": record.get("status"),
-                "summary": record.get("summary") or record.get("reason"),
-                "verdict": record.get("verdict"),
-                "paths": record.get("paths") or {},
-                "details": record.get("details") or {},
-                "metrics_summary": record.get("metrics_summary") or {},
-                "metric_rows": record.get("metric_rows") or [],
-                "metric_contract": record.get("metric_contract") or {},
-                "baseline_ref": record.get("baseline_ref") or {},
-                "baseline_comparisons": record.get("baseline_comparisons") or {},
-                "progress_eval": record.get("progress_eval") or {},
-                "evaluation_summary": record.get("evaluation_summary")
-                or ((record.get("details") or {}) if isinstance(record.get("details"), dict) else {}).get("evaluation_summary")
-                or {},
-                "files_changed": record.get("files_changed") or [],
-                "evidence_paths": record.get("evidence_paths") or [],
-                "updated_at": record.get("updated_at"),
-            }
-            state["breakthrough"] = bool(((record.get("progress_eval") or {}).get("breakthrough")))
-            state["breakthrough_level"] = ((record.get("progress_eval") or {}).get("breakthrough_level"))
+            candidate_sort_key = _run_result_sort_key(record, resolved_run_result, path)
+            current_sort_key = state.get("_latest_result_sort_key")
+            if current_sort_key is None or candidate_sort_key > current_sort_key:
+                state["_latest_result_sort_key"] = candidate_sort_key
+                state["latest_result"] = resolved_run_result
+                state["run_id"] = resolved_run_result.get("run_id") or state.get("run_id")
+                state["run_kind"] = resolved_run_result.get("run_kind") or state.get("run_kind")
+                progress_eval = (
+                    resolved_run_result.get("progress_eval")
+                    if isinstance(resolved_run_result.get("progress_eval"), dict)
+                    else {}
+                )
+                state["breakthrough"] = bool(progress_eval.get("breakthrough"))
+                state["breakthrough_level"] = progress_eval.get("breakthrough_level")
         if record.get("summary") or record.get("message") or record.get("reason"):
-            state["latest_summary"] = record.get("summary") or record.get("message") or record.get("reason")
+            current_summary_sort_key = state.get("_latest_summary_sort_key")
+            if current_summary_sort_key is None or artifact_sort_key > current_summary_sort_key:
+                state["_latest_summary_sort_key"] = artifact_sort_key
+                state["latest_summary"] = record.get("summary") or record.get("message") or record.get("reason")
         state["recent_artifacts"].append(
             {
                 "artifact_id": record.get("artifact_id"),
@@ -346,12 +342,107 @@ def _collect_branch_state(repo: Path) -> dict[str, dict[str, Any]]:
                 "reason": record.get("reason"),
                 "updated_at": record.get("updated_at"),
                 "status": record.get("status"),
+                "_sort_key": artifact_sort_key,
             }
         )
+        state["recent_artifacts"].sort(key=lambda item: item.get("_sort_key") or ("", 0, ""))
         state["recent_artifacts"] = state["recent_artifacts"][-4:]
+    for state in branch_state.values():
+        latest_result = state.get("latest_result")
+        if isinstance(latest_result, dict):
+            result_metric = extract_latest_metric(latest_result)
+            if result_metric is not None:
+                state["latest_metric"] = result_metric
+        for item in state.get("recent_artifacts", []):
+            if isinstance(item, dict):
+                item.pop("_sort_key", None)
     return branch_state
 
 
+def _resolve_run_result_payload(repo: Path, record: dict[str, Any]) -> dict[str, Any]:
+    details = dict(record.get("details") or {}) if isinstance(record.get("details"), dict) else {}
+    paths = dict(record.get("paths") or {}) if isinstance(record.get("paths"), dict) else {}
+    result_payload: dict[str, Any] = {}
+    result_json_path = _resolve_result_json_path(repo, paths.get("result_json"))
+    if result_json_path and result_json_path.exists():
+        loaded = read_json(result_json_path, {})
+        if isinstance(loaded, dict):
+            result_payload = loaded
+
+    evaluation_summary = (
+        record.get("evaluation_summary")
+        or details.get("evaluation_summary")
+        or result_payload.get("evaluation_summary")
+        or {}
+    )
+    progress_eval = record.get("progress_eval")
+    if not isinstance(progress_eval, dict):
+        progress_eval = result_payload.get("progress_eval") if isinstance(result_payload.get("progress_eval"), dict) else {}
+
+    return {
+        "run_id": record.get("run_id") or result_payload.get("run_id"),
+        "run_kind": record.get("run_kind") or result_payload.get("run_kind"),
+        "status": record.get("status") or result_payload.get("status"),
+        "summary": record.get("summary") or record.get("reason") or result_payload.get("summary"),
+        "verdict": record.get("verdict") or result_payload.get("verdict"),
+        "paths": paths or (result_payload.get("paths") if isinstance(result_payload.get("paths"), dict) else {}) or {},
+        "details": details,
+        "metrics_summary": record.get("metrics_summary") or result_payload.get("metrics_summary") or {},
+        "metric_rows": record.get("metric_rows") or result_payload.get("metric_rows") or [],
+        "metric_contract": record.get("metric_contract") or result_payload.get("metric_contract") or {},
+        "baseline_ref": record.get("baseline_ref") or result_payload.get("baseline_ref") or {},
+        "baseline_comparisons": record.get("baseline_comparisons") or result_payload.get("baseline_comparisons") or {},
+        "progress_eval": progress_eval or {},
+        "evaluation_summary": evaluation_summary or {},
+        "files_changed": record.get("files_changed") or result_payload.get("files_changed") or [],
+        "evidence_paths": record.get("evidence_paths") or result_payload.get("evidence_paths") or [],
+        "updated_at": record.get("updated_at") or result_payload.get("updated_at"),
+    }
+
+
+def _resolve_result_json_path(repo: Path, raw_path: object) -> Path | None:
+    normalized = str(raw_path or "").strip()
+    if not normalized:
+        return None
+    candidate = Path(normalized).expanduser()
+    if candidate.is_absolute():
+        return candidate
+    return repo / candidate
+
+
+def _artifact_record_sort_key(record: dict[str, Any], path: Path) -> tuple[str, int, str]:
+    updated_at = str(record.get("updated_at") or record.get("created_at") or "").strip()
+    try:
+        mtime_ns = path.stat().st_mtime_ns
+    except OSError:
+        mtime_ns = 0
+    return (updated_at, mtime_ns, str(path))
+
+
+def _run_result_sort_key(record: dict[str, Any], payload: dict[str, Any], path: Path) -> tuple[int, str, int, str]:
+    quality = 0
+    if extract_latest_metric(payload):
+        quality += 8
+    if payload.get("baseline_comparisons"):
+        quality += 4
+    if payload.get("progress_eval"):
+        quality += 4
+    if payload.get("metrics_summary"):
+        quality += 3
+    if payload.get("metric_rows"):
+        quality += 3
+    if payload.get("verdict"):
+        quality += 2
+    paths = payload.get("paths") if isinstance(payload.get("paths"), dict) else {}
+    if paths.get("result_json"):
+        quality += 2
+    if paths.get("run_md"):
+        quality += 1
+    updated_at = str(payload.get("updated_at") or record.get("updated_at") or record.get("created_at") or "")
+    _, mtime_ns, path_str = _artifact_record_sort_key(record, path)
+    return (quality, updated_at, mtime_ns, path_str)
+
+
 def _classify_ref(ref: str, state: dict[str, Any]) -> dict[str, str]:
     run_id = str(state.get("run_id") or "")
     run_kind = str(state.get("run_kind") or "")

package/src/deepscientist/lingzhu_support.py

@@ -1,182 +1 @@
-from __future__ import annotations
-
-import json
-import secrets
-from typing import Any
-from urllib.parse import urlparse
-
-
-DEFAULT_LINGZHU_GATEWAY_PORT = 18789
-DEFAULT_LINGZHU_LOCAL_HOST = "127.0.0.1"
-DEFAULT_LINGZHU_AGENT_ID = "main"
-DEFAULT_LINGZHU_SESSION_NAMESPACE = "lingzhu"
-
-_AUTH_AK_SEGMENTS = (8, 4, 4, 4, 12)
-_AUTH_AK_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789"
-
-
-def generate_lingzhu_auth_ak() -> str:
-    parts: list[str] = []
-    for segment_length in _AUTH_AK_SEGMENTS:
-        parts.append("".join(secrets.choice(_AUTH_AK_CHARS) for _ in range(segment_length)))
-    return "-".join(parts)
-
-
-def lingzhu_local_host(config: dict[str, Any] | None) -> str:
-    value = str((config or {}).get("local_host") or DEFAULT_LINGZHU_LOCAL_HOST).strip()
-    return value or DEFAULT_LINGZHU_LOCAL_HOST
-
-
-def lingzhu_gateway_port(config: dict[str, Any] | None) -> int:
-    raw = (config or {}).get("gateway_port")
-    try:
-        value = int(raw)
-    except (TypeError, ValueError):
-        return DEFAULT_LINGZHU_GATEWAY_PORT
-    if value < 1 or value > 65535:
-        return DEFAULT_LINGZHU_GATEWAY_PORT
-    return value
-
-
-def normalize_public_base_url(value: Any) -> str | None:
-    text = str(value or "").strip()
-    if not text:
-        return None
-    parsed = urlparse(text)
-    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
-        return None
-    return text.rstrip("/")
-
-
-def lingzhu_local_base_url(config: dict[str, Any] | None) -> str:
-    return f"http://{lingzhu_local_host(config)}:{lingzhu_gateway_port(config)}"
-
-
-def lingzhu_public_base_url(config: dict[str, Any] | None) -> str | None:
-    return normalize_public_base_url((config or {}).get("public_base_url"))
-
-
-def lingzhu_health_url(config: dict[str, Any] | None, *, public: bool = False) -> str | None:
-    base = lingzhu_public_base_url(config) if public else lingzhu_local_base_url(config)
-    if not base:
-        return None
-    return f"{base}/metis/agent/api/health"
-
-
-def lingzhu_sse_url(config: dict[str, Any] | None, *, public: bool = False) -> str | None:
-    base = lingzhu_public_base_url(config) if public else lingzhu_local_base_url(config)
-    if not base:
-        return None
-    return f"{base}/metis/agent/api/sse"
-
-
-def lingzhu_agent_id(config: dict[str, Any] | None) -> str:
-    value = str((config or {}).get("agent_id") or DEFAULT_LINGZHU_AGENT_ID).strip()
-    return value or DEFAULT_LINGZHU_AGENT_ID
-
-
-def lingzhu_probe_payload(
-    config: dict[str, Any] | None,
-    *,
-    message_id: str = "ds-lingzhu-probe-001",
-    text: str = "你好",
-) -> dict[str, Any]:
-    return {
-        "message_id": message_id,
-        "agent_id": lingzhu_agent_id(config),
-        "message": [
-            {
-                "role": "user",
-                "type": "text",
-                "text": text,
-            }
-        ],
-    }
-
-
-def lingzhu_generated_openclaw_config(config: dict[str, Any] | None) -> dict[str, Any]:
-    resolved = dict(config or {})
-    return {
-        "gateway": {
-            "port": lingzhu_gateway_port(resolved),
-            "http": {
-                "endpoints": {
-                    "chatCompletions": {
-                        "enabled": True,
-                    }
-                }
-            },
-        },
-        "plugins": {
-            "entries": {
-                "lingzhu": {
-                    "enabled": bool(resolved.get("enabled", False)),
-                    "config": {
-                        "authAk": str(resolved.get("auth_ak") or "").strip(),
-                        "agentId": lingzhu_agent_id(resolved),
-                        "includeMetadata": bool(resolved.get("include_metadata", True)),
-                        "requestTimeoutMs": int(resolved.get("request_timeout_ms") or 60000),
-                        "systemPrompt": str(resolved.get("system_prompt") or ""),
-                        "defaultNavigationMode": str(resolved.get("default_navigation_mode") or "0"),
-                        "enableFollowUp": bool(resolved.get("enable_follow_up", True)),
-                        "followUpMaxCount": int(resolved.get("follow_up_max_count") or 3),
-                        "maxImageBytes": int(resolved.get("max_image_bytes") or 5 * 1024 * 1024),
-                        "sessionMode": str(resolved.get("session_mode") or "per_user"),
-                        "sessionNamespace": str(
-                            resolved.get("session_namespace") or DEFAULT_LINGZHU_SESSION_NAMESPACE
-                        ),
-                        "autoReceiptAck": bool(resolved.get("auto_receipt_ack", True)),
-                        "visibleProgressHeartbeat": bool(
-                            resolved.get("visible_progress_heartbeat", True)
-                        ),
-                        "visibleProgressHeartbeatSec": int(
-                            resolved.get("visible_progress_heartbeat_sec") or 10
-                        ),
-                        "debugLogging": bool(resolved.get("debug_logging", False)),
-                        "debugLogPayloads": bool(resolved.get("debug_log_payloads", False)),
-                        "debugLogDir": str(resolved.get("debug_log_dir") or ""),
-                        "enableExperimentalNativeActions": bool(
-                            resolved.get("enable_experimental_native_actions", False)
-                        ),
-                    },
-                }
-            }
-        },
-    }
-
-
-def lingzhu_generated_openclaw_config_text(config: dict[str, Any] | None) -> str:
-    return json.dumps(lingzhu_generated_openclaw_config(config), indent=2, ensure_ascii=False)
-
-
-def lingzhu_generated_curl(config: dict[str, Any] | None, *, text: str = "你好") -> str:
-    auth_ak = str((config or {}).get("auth_ak") or "").strip()
-    payload = lingzhu_probe_payload(config, text=text)
-    endpoint_url = lingzhu_sse_url(config) or ""
-    return (
-        f"curl -X POST '{endpoint_url}' \\\n"
-        f"  --header 'Authorization: Bearer {auth_ak}' \\\n"
-        "  --header 'Content-Type: application/json' \\\n"
-        f"  --data '{json.dumps(payload, ensure_ascii=False)}'"
-    )
-
-
-def lingzhu_supported_commands(*, experimental_enabled: bool) -> list[str]:
-    commands = [
-        "take_photo",
-        "take_navigation",
-        "control_calendar",
-        "notify_agent_off",
-    ]
-    if experimental_enabled:
-        commands.extend(
-            [
-                "send_notification",
-                "send_toast",
-                "speak_tts",
-                "start_video_record",
-                "stop_video_record",
-                "open_custom_view",
-            ]
-        )
-    return commands
+from .connector.lingzhu_support import *  # noqa: F401,F403

package/src/deepscientist/mcp/server.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 import argparse
+from collections import deque
+from pathlib import Path
 from typing import Any
 
 from mcp.server.fastmcp import FastMCP
@@ -95,6 +97,125 @@ def _build_default_bash_log_payload(log_text: str) -> dict[str, Any]:
     }
 
 
+def _stream_bash_log_summary(path: Path) -> tuple[list[str], int, list[str]]:
+    total = 0
+    full_lines: list[str] = []
+    head_lines: list[str] = []
+    tail_lines: deque[str] = deque(maxlen=DEFAULT_INLINE_BASH_LOG_TAIL_LINES)
+    with path.open("r", encoding="utf-8", errors="replace") as handle:
+        for raw_line in handle:
+            line = raw_line.rstrip("\n")
+            total += 1
+            if total <= DEFAULT_INLINE_BASH_LOG_LINE_LIMIT:
+                full_lines.append(line)
+                continue
+            if total == DEFAULT_INLINE_BASH_LOG_LINE_LIMIT + 1:
+                head_lines = full_lines[:DEFAULT_INLINE_BASH_LOG_HEAD_LINES]
+                tail_lines.extend(full_lines[-DEFAULT_INLINE_BASH_LOG_TAIL_LINES :])
+                full_lines = []
+            tail_lines.append(line)
+    return full_lines, total, list(head_lines or tail_lines)
+
+
+def _build_default_bash_log_payload_from_path(path: Path) -> dict[str, Any]:
+    if not path.exists():
+        return {
+            "log": "",
+            "log_line_count": 0,
+            "log_truncated": False,
+        }
+    full_lines, total, preview_seed = _stream_bash_log_summary(path)
+    if total <= DEFAULT_INLINE_BASH_LOG_LINE_LIMIT:
+        return {
+            "log": _join_bash_log_lines(full_lines),
+            "log_line_count": total,
+            "log_truncated": False,
+        }
+    with path.open("r", encoding="utf-8", errors="replace") as handle:
+        tail_lines: deque[str] = deque(maxlen=DEFAULT_INLINE_BASH_LOG_TAIL_LINES)
+        for raw_line in handle:
+            tail_lines.append(raw_line.rstrip("\n"))
+    omitted = total - DEFAULT_INLINE_BASH_LOG_HEAD_LINES - DEFAULT_INLINE_BASH_LOG_TAIL_LINES
+    marker = (
+        f"[... omitted {omitted} lines from the middle of this log. {LONG_BASH_LOG_HINT}]"
+    )
+    preview_lines = preview_seed[:DEFAULT_INLINE_BASH_LOG_HEAD_LINES] + [marker] + list(tail_lines)
+    return {
+        "log": _join_bash_log_lines(preview_lines),
+        "log_line_count": total,
+        "log_truncated": True,
+        "log_preview_head_lines": DEFAULT_INLINE_BASH_LOG_HEAD_LINES,
+        "log_preview_tail_lines": DEFAULT_INLINE_BASH_LOG_TAIL_LINES,
+        "log_preview_omitted_lines": omitted,
+        "log_read_hint": LONG_BASH_LOG_HINT,
+    }
+
+
+def _build_bash_log_window_from_path(path: Path, *, start: int | None = None, tail: int | None = None) -> dict[str, Any]:
+    if not path.exists():
+        return {
+            "log": "",
+            "log_line_count": 0,
+            "log_windowed": True,
+            "line_start": 1,
+            "line_end": 0,
+            "line_limit": _normalize_bash_log_window_size(tail),
+            "returned_line_count": 0,
+            "has_more_before": False,
+            "has_more_after": False,
+            "log_read_hint": LONG_BASH_LOG_HINT,
+        }
+    line_limit = _normalize_bash_log_window_size(tail)
+    if start is not None:
+        requested_start = max(1, int(start))
+        selected: list[str] = []
+        total = 0
+        with path.open("r", encoding="utf-8", errors="replace") as handle:
+            for raw_line in handle:
+                total += 1
+                if total < requested_start:
+                    continue
+                if len(selected) < line_limit:
+                    selected.append(raw_line.rstrip("\n"))
+        returned_count = len(selected)
+        line_start = requested_start if total else 1
+        line_end = requested_start + returned_count - 1 if returned_count else requested_start - 1
+        return {
+            "log": _join_bash_log_lines(selected),
+            "log_line_count": total,
+            "log_windowed": True,
+            "line_start": line_start,
+            "line_end": line_end,
+            "line_limit": line_limit,
+            "returned_line_count": returned_count,
+            "has_more_before": line_start > 1,
+            "has_more_after": line_end < total,
+            "log_read_hint": LONG_BASH_LOG_HINT,
+        }
+
+    tail_lines: deque[str] = deque(maxlen=line_limit)
+    total = 0
+    with path.open("r", encoding="utf-8", errors="replace") as handle:
+        for raw_line in handle:
+            total += 1
+            tail_lines.append(raw_line.rstrip("\n"))
+    returned_count = len(tail_lines)
+    line_start = max(1, total - returned_count + 1) if total else 1
+    line_end = total
+    return {
+        "log": _join_bash_log_lines(list(tail_lines)),
+        "log_line_count": total,
+        "log_windowed": True,
+        "line_start": line_start,
+        "line_end": line_end,
+        "line_limit": line_limit,
+        "returned_line_count": returned_count,
+        "has_more_before": line_start > 1,
+        "has_more_after": False,
+        "log_read_hint": LONG_BASH_LOG_HINT,
+    }
+
+
 def build_memory_server(context: McpContext) -> FastMCP:
     service = MemoryService(context.home)
     server = FastMCP(
@@ -680,16 +801,23 @@ def build_artifact_server(context: McpContext) -> FastMCP:
     @server.tool(
         name="arxiv",
         description=(
-            "Read an identified arXiv paper by id. "
-            "Use full_text=false for the overview/abstract path and full_text=true when the full paper body is needed."
+            "Interact with the quest-local arXiv library. "
+            "Use mode='read' to read one paper by id with local-first automatic persistence, "
+            "or mode='list' to list the saved arXiv items for the current quest."
         ),
     )
     def arxiv(
-        paper_id: str,
+        paper_id: str | None = None,
+        mode: str = "read",
         full_text: bool = False,
         comment: str | dict[str, Any] | None = None,
     ) -> dict[str, Any]:
-        return service.arxiv(paper_id, full_text=full_text)
+        return service.arxiv(
+            paper_id,
+            mode=mode,
+            full_text=full_text,
+            quest_root=context.require_quest_root(),
+        )
 
     @server.tool(name="refresh_summary", description="Refresh SUMMARY.md from recent artifact state.")
     def refresh_summary(
@@ -873,8 +1001,8 @@ def build_bash_exec_server(context: McpContext) -> FastMCP:
                     export_log_to=export_log_to,
                 )
                 payload.update(
-                    _build_bash_log_window(
-                        service.read_terminal_log(quest_root, bash_id),
+                    _build_bash_log_window_from_path(
+                        service.terminal_log_path(quest_root, bash_id),
                         start=start,
                         tail=tail if tail is not None else tail_limit,
                     )
@@ -914,7 +1042,7 @@ def build_bash_exec_server(context: McpContext) -> FastMCP:
                 export_log=export_log,
                 export_log_to=export_log_to,
             )
-            payload.update(_build_default_bash_log_payload(service.read_terminal_log(quest_root, bash_id)))
+            payload.update(_build_default_bash_log_payload_from_path(service.terminal_log_path(quest_root, bash_id)))
             return payload
         if normalized_mode == "kill":
             bash_id = service.resolve_session_id(quest_root, id)