@researai/deepscientist 1.5.7 → 1.5.9

package/src/deepscientist/daemon/app.py

@@ -39,6 +39,7 @@ from ..connector_profiles import (
 )
 from ..connector_runtime import conversation_identity_key, format_conversation_id, normalize_conversation_id, parse_conversation_id
 from ..config import ConfigManager
+from ..config.models import SYSTEM_CONNECTOR_NAMES
 from ..home import repo_root
 from ..memory import MemoryService
 from ..network import urlopen_with_proxy as urlopen
@@ -80,6 +81,7 @@ class DaemonApp:
         self.daemon_managed_by = str(os.environ.get("DS_DAEMON_MANAGED_BY") or "manual").strip() or "manual"
         self.repo_root = repo_root()
         self.config_manager = ConfigManager(home)
+        self.runtime_config = self.config_manager.load_runtime_config()
         self.runners_config = self.config_manager.load_runners_config()
         self.connectors_config = self.config_manager.load_named_normalized("connectors")
         self.skill_installer = SkillInstaller(self.repo_root, home)
@@ -90,7 +92,7 @@ class DaemonApp:
         self.bash_exec_service = BashExecService(home)
         self.team_service = SingleTeamService(home)
         self.cloud_service = CloudLinkService(home)
-        config = self.config_manager.load_named("config")
+        config = self.runtime_config
         skill_config = config.get("skills") if isinstance(config.get("skills"), dict) else {}
         self.skill_sync_summary = self.skill_installer.ensure_release_sync(
             installed_version=__version__,
@@ -145,9 +147,13 @@ class DaemonApp:
 
     def list_connector_statuses(self) -> list[dict[str, object]]:
         title_by_quest = self._quest_titles_by_id()
-        items = [self._augment_connector_status(channel.status(), title_by_quest=title_by_quest) for channel in self.channels.values()]
+        items = [
+            self._augment_connector_status(channel.status(), title_by_quest=title_by_quest)
+            for name, channel in self.channels.items()
+            if name == "local" or self._is_connector_system_enabled(name)
+        ]
         lingzhu_config = self.connectors_config.get("lingzhu")
-        if isinstance(lingzhu_config, dict):
+        if isinstance(lingzhu_config, dict) and self._is_connector_system_enabled("lingzhu"):
             items.append(self._augment_connector_status(self.config_manager.lingzhu_snapshot(lingzhu_config), title_by_quest=title_by_quest))
         return items
 
@@ -346,6 +352,8 @@ class DaemonApp:
                 continue
             if connector_name not in self.channels:
                 continue
+            if not self._is_connector_system_enabled(connector_name):
+                continue
             if parsed is not None and str(parsed.get("connector") or "").strip().lower() != connector_name:
                 continue
             normalized_by_connector[connector_name] = {
@@ -459,19 +467,203 @@ class DaemonApp:
                 b"Terminal attach token is invalid or expired.",
             )
         if runtime is None:
-            return Response(
-                409,
-                "Conflict",
-                Headers({"Content-Type": "text/plain; charset=utf-8"}),
-                b"Terminal runtime is no longer active.",
-            )
+            try:
+                session = self.bash_exec_service.get_session(attach_token.quest_root, attach_token.bash_id)
+            except FileNotFoundError:
+                return Response(
+                    404,
+                    "Not Found",
+                    Headers({"Content-Type": "text/plain; charset=utf-8"}),
+                    b"Terminal session is no longer available.",
+                )
+            status = str(session.get("status") or "").strip().lower()
+            kind = str(session.get("kind") or "").strip().lower()
+            if status in {"completed", "failed", "terminated"} or kind not in {"exec"}:
+                return Response(
+                    409,
+                    "Conflict",
+                    Headers({"Content-Type": "text/plain; charset=utf-8"}),
+                    b"Terminal runtime is no longer active.",
+                )
         setattr(connection, "_ds_terminal_attach_token", token)
         return None
 
+    def _handle_logged_terminal_attach_connection(
+        self,
+        connection: ServerConnection,
+        *,
+        attach_token,
+        send_lock: threading.Lock,
+    ) -> None:
+        session = self.bash_exec_service.get_session(attach_token.quest_root, attach_token.bash_id)
+        stop_event = threading.Event()
+
+        with send_lock:
+            connection.send(
+                json.dumps(
+                    {
+                        "type": "ready",
+                        "bash_id": attach_token.bash_id,
+                        "status": session.get("status"),
+                        "cwd": session.get("cwd"),
+                        "workdir": session.get("workdir"),
+                    },
+                    ensure_ascii=False,
+                )
+            )
+
+        def _encode_exec_log_entry(entry: dict[str, Any]) -> bytes:
+            line = str(entry.get("line") or "")
+            stream = str(entry.get("stream") or "").strip().lower()
+            if line.startswith("__DS_PROGRESS__") or line.startswith("__DS_BASH_STATUS__"):
+                return b""
+            if line.startswith("__DS_BASH_CR__"):
+                payload = line[len("__DS_BASH_CR__") :].lstrip()
+                return f"\r\x1b[K{payload}".encode("utf-8", errors="replace")
+            if stream in {"prompt", "partial"}:
+                return line.encode("utf-8", errors="replace")
+            if stream == "system" and not line.strip():
+                return b""
+            return f"{line}\n".encode("utf-8", errors="replace")
+
+        def _relay_output() -> None:
+            last_seq = 0
+            try:
+                entries, meta = self.bash_exec_service.read_log_entries(
+                    attach_token.quest_root,
+                    attach_token.bash_id,
+                    limit=2000,
+                    order="asc",
+                )
+                if isinstance(meta.get("latest_seq"), int):
+                    last_seq = int(meta["latest_seq"])
+                elif entries:
+                    last_seq = max(int(item.get("seq") or 0) for item in entries)
+                for entry in entries:
+                    payload = _encode_exec_log_entry(entry)
+                    if not payload:
+                        continue
+                    with send_lock:
+                        connection.send(payload)
+
+                while not stop_event.is_set():
+                    entries, _meta = self.bash_exec_service.read_log_entries(
+                        attach_token.quest_root,
+                        attach_token.bash_id,
+                        limit=400,
+                        after_seq=last_seq,
+                        order="asc",
+                    )
+                    for entry in entries:
+                        last_seq = max(last_seq, int(entry.get("seq") or 0))
+                        payload = _encode_exec_log_entry(entry)
+                        if not payload:
+                            continue
+                        with send_lock:
+                            connection.send(payload)
+                    current = self.bash_exec_service.get_session(
+                        attach_token.quest_root,
+                        attach_token.bash_id,
+                    )
+                    status = str(current.get("status") or "").strip().lower()
+                    if status in {"completed", "failed", "terminated"}:
+                        with send_lock:
+                            connection.send(
+                                json.dumps(
+                                    {
+                                        "type": "exit",
+                                        "bash_id": attach_token.bash_id,
+                                        "status": current.get("status"),
+                                        "exit_code": current.get("exit_code"),
+                                        "stop_reason": current.get("stop_reason"),
+                                        "finished_at": current.get("finished_at"),
+                                    },
+                                    ensure_ascii=False,
+                                )
+                            )
+                        return
+                    time.sleep(TERMINAL_STREAM_IDLE_SLEEP_SECONDS)
+            except Exception as exc:
+                if stop_event.is_set():
+                    return
+                try:
+                    with send_lock:
+                        connection.send(
+                            json.dumps({"type": "error", "message": str(exc)}, ensure_ascii=False)
+                        )
+                except Exception:
+                    pass
+
+        relay_thread = threading.Thread(
+            target=_relay_output,
+            daemon=True,
+            name=f"exec-attach-{attach_token.bash_id}",
+        )
+        relay_thread.start()
+        try:
+            while True:
+                try:
+                    message = connection.recv()
+                except ConnectionClosed:
+                    break
+                if message is None:
+                    break
+                if isinstance(message, bytes):
+                    self.bash_exec_service.append_terminal_input(
+                        attach_token.quest_root,
+                        attach_token.bash_id,
+                        data=message.decode("utf-8", errors="replace"),
+                        source="web-pty",
+                    )
+                    continue
+                try:
+                    payload = json.loads(message)
+                except json.JSONDecodeError:
+                    continue
+                if not isinstance(payload, dict):
+                    continue
+                message_type = str(payload.get("type") or "").strip().lower()
+                if message_type == "input":
+                    self.bash_exec_service.append_terminal_input(
+                        attach_token.quest_root,
+                        attach_token.bash_id,
+                        data=str(payload.get("data") or ""),
+                        source="web-pty",
+                    )
+                    continue
+                if message_type == "binary_input":
+                    raw = str(payload.get("data") or "")
+                    if raw:
+                        self.bash_exec_service.append_terminal_input(
+                            attach_token.quest_root,
+                            attach_token.bash_id,
+                            data=base64.b64decode(raw).decode("utf-8", errors="replace"),
+                            source="web-pty",
+                        )
+                    continue
+                if message_type == "resize":
+                    cols = int(payload.get("cols") or 0)
+                    rows = int(payload.get("rows") or 0)
+                    self.bash_exec_service.resize_terminal_session(
+                        attach_token.quest_root,
+                        attach_token.bash_id,
+                        cols=cols,
+                        rows=rows,
+                    )
+                    continue
+                if message_type == "detach":
+                    break
+                if message_type == "ping":
+                    with send_lock:
+                        connection.send(json.dumps({"type": "pong"}, ensure_ascii=False))
+        finally:
+            stop_event.set()
+            relay_thread.join(timeout=1)
+
     def _handle_terminal_attach_connection(self, connection: ServerConnection) -> None:
         token_value = str(getattr(connection, "_ds_terminal_attach_token", "") or "").strip()
         attach_token, runtime = self.bash_exec_service.consume_terminal_attach_token(token_value)
-        if attach_token is None or runtime is None:
+        if attach_token is None:
             try:
                 connection.close(code=1011, reason="terminal_attach_unavailable")
             except Exception:
@@ -479,6 +671,31 @@ class DaemonApp:
             return
 
         send_lock = threading.Lock()
+        if runtime is None:
+            try:
+                self._handle_logged_terminal_attach_connection(
+                    connection,
+                    attach_token=attach_token,
+                    send_lock=send_lock,
+                )
+            except Exception as exc:
+                try:
+                    with send_lock:
+                        connection.send(
+                            json.dumps(
+                                {"type": "error", "message": str(exc)},
+                                ensure_ascii=False,
+                            )
+                        )
+                except Exception:
+                    pass
+            finally:
+                try:
+                    connection.close()
+                except Exception:
+                    pass
+            return
+
         client = TerminalClient(
             client_id=generate_id("tclient"),
             send_text=connection.send,
@@ -635,6 +852,43 @@ class DaemonApp:
         factory = get_channel_factory(name)
         return factory(home=self.home, app=self, config=self.connectors_config.get(name, {}))
 
+    def _system_enabled_connector_names(self) -> set[str]:
+        connectors = self.runtime_config.get("connectors") if isinstance(self.runtime_config.get("connectors"), dict) else {}
+        system_enabled = connectors.get("system_enabled") if isinstance(connectors.get("system_enabled"), dict) else {}
+        return {
+            name
+            for name in SYSTEM_CONNECTOR_NAMES
+            if bool(system_enabled.get(name, name == "qq"))
+        }
+
+    def _is_connector_system_enabled(self, connector_name: str) -> bool:
+        normalized = str(connector_name or "").strip().lower()
+        if normalized == "local":
+            return True
+        enabled = self._system_enabled_connector_names()
+        if normalized in enabled:
+            return True
+        if normalized in SYSTEM_CONNECTOR_NAMES:
+            return False
+        return True
+
+    def reload_runtime_config(self, *, restart_background: bool = True) -> dict[str, object]:
+        previous_enabled = self._system_enabled_connector_names()
+        self.runtime_config = self.config_manager.load_runtime_config()
+        logging_config = self.runtime_config.get("logging") if isinstance(self.runtime_config.get("logging"), dict) else {}
+        self.logger.level = str(logging_config.get("level") or "info").strip().lower() or "info"
+        enabled = self._system_enabled_connector_names()
+        restarted = False
+        if restart_background and self._server is not None and enabled != previous_enabled:
+            self._stop_background_connectors()
+            self._start_background_connectors()
+            restarted = True
+        return {
+            "ok": True,
+            "system_enabled_connectors": sorted(enabled),
+            "restarted_background_connectors": restarted,
+        }
+
     def reload_connectors_config(self, *, restart_background: bool = True) -> dict[str, object]:
         self.connectors_config = self.config_manager.load_named_normalized("connectors")
         register_builtin_channels(home=self.home, connectors_config=self.connectors_config)
@@ -650,7 +904,11 @@ class DaemonApp:
         return {
             "ok": True,
             "connectors": sorted(
-                name for name, config in self.connectors_config.items() if not str(name).startswith("_") and isinstance(config, dict)
+                name
+                for name, config in self.connectors_config.items()
+                if not str(name).startswith("_")
+                and isinstance(config, dict)
+                and self._is_connector_system_enabled(str(name))
             ),
         }
 
@@ -671,8 +929,7 @@ class DaemonApp:
         }
 
     def _preferred_locale(self) -> str:
-        config = self.config_manager.load_named("config")
-        return str(config.get("default_locale") or "en-US").lower()
+        return str(self.runtime_config.get("default_locale") or "en-US").lower()
 
     def _polite_copy(self, *, zh: str, en: str) -> str:
         return zh if self._preferred_locale().startswith("zh") else en
@@ -741,7 +998,7 @@ class DaemonApp:
         exclude_conversation_id: str | None = None,
         preferred_connector_conversation_id: str | None = None,
         requested_connector_bindings: list[dict[str, object]] | None = None,
-        force_connector_rebind: bool = False,
+        force_connector_rebind: bool = True,
         requested_baseline_ref: dict[str, object] | None = None,
         startup_contract: dict[str, object] | None = None,
     ) -> dict:
@@ -2666,6 +2923,13 @@ class DaemonApp:
         }
 
     def handle_connector_inbound(self, connector_name: str, body: dict) -> dict:
+        if not self._is_connector_system_enabled(connector_name):
+            return {
+                "ok": True,
+                "accepted": False,
+                "reason": "system_disabled",
+                "message": f"Connector `{connector_name}` is disabled at the system level.",
+            }
         channel = self._channel_with_bindings(connector_name)
         ingested = channel.ingest(body)
         if not ingested.get("accepted", False):
@@ -3330,6 +3594,8 @@ class DaemonApp:
         for connector_name, channel in self.channels.items():
             if connector_name == "local":
                 continue
+            if not self._is_connector_system_enabled(connector_name):
+                continue
             connector_config = self.connectors_config.get(connector_name, {})
             if not isinstance(connector_config, dict):
                 continue
@@ -3523,6 +3789,8 @@ class DaemonApp:
         }
 
     def _maybe_auto_bind_connector_conversation(self, connector_name: str, conversation_id: str) -> dict | None:
+        if not self._is_connector_system_enabled(connector_name):
+            return None
         connector_config = self.connectors_config.get(connector_name, {})
         if not isinstance(connector_config, dict):
             return None
@@ -3746,6 +4014,8 @@ class DaemonApp:
         )
 
     def _profiled_connector_configs(self, connector_name: str) -> list[tuple[str, str | None, dict[str, Any]]]:
+        if not self._is_connector_system_enabled(connector_name):
+            return []
         connector_config = self.connectors_config.get(connector_name, {})
         if not isinstance(connector_config, dict):
             return []
@@ -3763,7 +4033,7 @@ class DaemonApp:
 
     def _start_background_connectors(self) -> None:
         qq_config = self.connectors_config.get("qq", {})
-        if isinstance(qq_config, dict) and not self._qq_gateways:
+        if self._is_connector_system_enabled("qq") and isinstance(qq_config, dict) and not self._qq_gateways:
             profiles = list_qq_profiles(qq_config)
             encode_profile_id = len(profiles) > 1
             for profile in profiles:
@@ -3783,7 +4053,7 @@ class DaemonApp:
                 )
                 if gateway.start():
                     self._qq_gateways[profile_id] = gateway
-        if not self._telegram_polling:
+        if self._is_connector_system_enabled("telegram") and not self._telegram_polling:
             for profile_id, profile_label, profile_config in self._profiled_connector_configs("telegram"):
                 polling = TelegramPollingService(
                     home=self.home,
@@ -3801,7 +4071,7 @@ class DaemonApp:
                 )
                 if polling.start():
                     self._telegram_polling[profile_id] = polling
-        if not self._slack_socket:
+        if self._is_connector_system_enabled("slack") and not self._slack_socket:
             for profile_id, profile_label, profile_config in self._profiled_connector_configs("slack"):
                 slack = SlackSocketModeService(
                     home=self.home,
@@ -3819,7 +4089,7 @@ class DaemonApp:
                 )
                 if slack.start():
                     self._slack_socket[profile_id] = slack
-        if not self._discord_gateway:
+        if self._is_connector_system_enabled("discord") and not self._discord_gateway:
             for profile_id, profile_label, profile_config in self._profiled_connector_configs("discord"):
                 discord = DiscordGatewayService(
                     home=self.home,
@@ -3837,7 +4107,7 @@ class DaemonApp:
                 )
                 if discord.start():
                     self._discord_gateway[profile_id] = discord
-        if not self._feishu_long_connection:
+        if self._is_connector_system_enabled("feishu") and not self._feishu_long_connection:
             for profile_id, profile_label, profile_config in self._profiled_connector_configs("feishu"):
                 feishu = FeishuLongConnectionService(
                     home=self.home,
@@ -3855,7 +4125,7 @@ class DaemonApp:
                 )
                 if feishu.start():
                     self._feishu_long_connection[profile_id] = feishu
-        if not self._whatsapp_local_session:
+        if self._is_connector_system_enabled("whatsapp") and not self._whatsapp_local_session:
             for profile_id, profile_label, profile_config in self._profiled_connector_configs("whatsapp"):
                 whatsapp = WhatsAppLocalSessionService(
                     home=self.home,
@@ -4486,6 +4756,7 @@ class DaemonApp:
                         "git_commit",
                         "git_diff_file",
                         "git_commit_file",
+                        "file_change_diff",
                         "explorer",
                         "quest_search",
                         "node_traces",
@@ -4498,7 +4769,7 @@ class DaemonApp:
                         payload = result(**params, path=self.path)
                     elif method == "GET":
                         payload = result(**params) if params else result()
-                    elif route_name in {"document_open", "document_asset_upload", "chat", "command", "quest_control", "config_save", "quest_create", "quest_baseline_binding", "run_create", "qq_inbound", "connector_inbound", "docs_open", "admin_shutdown", "bash_stop", "quest_settings", "quest_bindings", "quest_delete", "terminal_session_ensure", "terminal_attach", "terminal_input", "stage_view", "latex_init", "latex_compile", "system_update_action"}:
+                    elif route_name in {"document_open", "document_asset_upload", "chat", "command", "quest_control", "config_save", "quest_create", "quest_baseline_binding", "run_create", "qq_inbound", "connector_inbound", "docs_open", "admin_shutdown", "bash_stop", "quest_settings", "quest_bindings", "quest_delete", "quest_layout_update", "terminal_session_ensure", "terminal_attach", "terminal_input", "stage_view", "latex_init", "latex_compile", "system_update_action"}:
                         payload = result(**params, body=body)
                     elif route_name == "config_validate":
                         payload = result(body)

package/src/deepscientist/gitops/diff.py

@@ -4,6 +4,7 @@ from collections import defaultdict
 from pathlib import Path
 from typing import Any
 
+from ..artifact.metrics import extract_latest_metric
 from ..shared import read_json, run_command
 from .service import branch_exists, current_branch, head_commit
 
@@ -308,16 +309,9 @@ def _collect_branch_state(repo: Path) -> dict[str, dict[str, Any]]:
             state["parent_branch"] = record.get("parent_branch")
         if record.get("worktree_root"):
             state["worktree_root"] = record.get("worktree_root")
-        if isinstance(record.get("metrics_summary"), dict) and record["metrics_summary"]:
-            key = next(iter(record["metrics_summary"]))
-            state["latest_metric"] = {
-                "key": ((record.get("progress_eval") or {}).get("primary_metric_id")) or key,
-                "value": record["metrics_summary"].get(
-                    ((record.get("progress_eval") or {}).get("primary_metric_id")) or key,
-                    record["metrics_summary"].get(key),
-                ),
-                "delta_vs_baseline": ((record.get("progress_eval") or {}).get("delta_vs_baseline")),
-            }
+        latest_metric = extract_latest_metric(record)
+        if latest_metric is not None:
+            state["latest_metric"] = latest_metric
         if record.get("kind") == "run":
             state["latest_result"] = {
                 "run_id": record.get("run_id"),
@@ -365,6 +359,8 @@ def _classify_ref(ref: str, state: dict[str, Any]) -> dict[str, str]:
         return {"branch_kind": "quest", "tier": "major", "mode": "ideas"}
     if ref.startswith("idea/"):
         return {"branch_kind": "idea", "tier": "major", "mode": "ideas"}
+    if ref.startswith("paper/"):
+        return {"branch_kind": "paper", "tier": "major", "mode": "ideas"}
     if ref.startswith("analysis/") or run_id.startswith("analysis") or run_kind == "analysis-campaign":
         return {"branch_kind": "analysis", "tier": "minor", "mode": "analysis"}
     return {"branch_kind": "implementation", "tier": "major", "mode": "ideas"}