@researai/deepscientist 1.5.1 → 1.5.3

package/src/deepscientist/bash_exec/monitor.py

@@ -202,6 +202,18 @@ def _terminate_process(process: subprocess.Popen[bytes], process_group_id: int |
         process.kill()
 
 
+def _terminate_process_force(process: subprocess.Popen[bytes], process_group_id: int | None) -> None:
+    if process.poll() is not None:
+        return
+    if isinstance(process_group_id, int) and process_group_id > 0:
+        try:
+            os.killpg(process_group_id, signal.SIGKILL)
+        except ProcessLookupError:
+            return
+    else:
+        process.kill()
+
+
 def _drain_buffer(
     buffer: str,
     append_line,
@@ -350,12 +362,15 @@ def run_monitor(session_dir: Path) -> int:
             },
         )
         progress = _parse_progress_marker(line)
+        output_updates: dict[str, Any] = {}
+        if stream not in {"system", "prompt"}:
+            output_updates = {"last_output_at": timestamp, "last_output_seq": seq}
         if progress is not None:
             progress.setdefault("ts", timestamp)
             _atomic_write_json(progress_path, progress)
-            update_meta(last_progress=progress, latest_seq=seq)
+            update_meta(last_progress=progress, latest_seq=seq, **output_updates)
         else:
-            update_meta(latest_seq=seq)
+            update_meta(latest_seq=seq, **output_updates)
 
     master_fd: int | None = None
     slave_fd: int | None = None
@@ -410,16 +425,20 @@ def run_monitor(session_dir: Path) -> int:
             if not stop_requested and stop_request_path.exists():
                 request = read_json(stop_request_path, {}) or {}
                 stop_reason = str(request.get("reason") or "user_stop").strip() or "user_stop"
+                force_stop = bool(request.get("force"))
                 update_meta(
                     status="terminating",
                     stop_reason=stop_reason,
                     stopped_by_user_id=str(request.get("user_id") or meta.get("stopped_by_user_id") or meta.get("agent_id") or "agent"),
                 )
                 append_line(
-                    f"Termination requested: {stop_reason}",
+                    f"{'Force t' if force_stop else 'T'}ermination requested: {stop_reason}",
                     stream="system",
                 )
-                _terminate_process(process, process_group_id)
+                if force_stop:
+                    _terminate_process_force(process, process_group_id)
+                else:
+                    _terminate_process(process, process_group_id)
                 stop_requested = True
 
             if deadline is not None and time.monotonic() >= deadline and process.poll() is None and not stop_requested:

package/src/deepscientist/bash_exec/runtime.py

@@ -390,6 +390,9 @@ class TerminalRuntime:
             progress.setdefault("ts", timestamp)
             _atomic_write_json(self.meta_path.parent / "progress.json", progress)
             meta["last_progress"] = progress
+        if stream not in {"system", "prompt"}:
+            meta["last_output_at"] = timestamp
+            meta["last_output_seq"] = seq
         meta["latest_seq"] = seq
         meta["updated_at"] = timestamp
         _atomic_write_json(self.meta_path, meta)

package/src/deepscientist/bash_exec/service.py

@@ -11,6 +11,7 @@ import sys
 import tempfile
 import threading
 import time
+from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
 
@@ -26,6 +27,7 @@ DEFAULT_LOG_TAIL_LIMIT = 200
 DEFAULT_POLL_INTERVAL_SECONDS = 0.35
 TERMINAL_STATUSES = {"completed", "failed", "terminated"}
 DEFAULT_TERMINAL_SESSION_ID = "terminal-main"
+BASH_WATCHDOG_AFTER_SECONDS = 1800
 INPUT_ESCAPE_SEQUENCE_RE = re.compile(r"\x1b\[[0-9;?]*[ -/]*[@-~]|\x1b[@-_]")
 
 
@@ -89,6 +91,48 @@ def _parse_progress_marker(line: str) -> dict[str, Any] | None:
     return payload
 
 
+def _parse_timestamp(value: object) -> datetime | None:
+    normalized = _normalize_string(value)
+    if not normalized:
+        return None
+    try:
+        parsed = datetime.fromisoformat(normalized)
+    except ValueError:
+        return None
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=UTC)
+    return parsed.astimezone(UTC)
+
+
+def _age_seconds(value: object, *, now: datetime | None = None) -> int | None:
+    parsed = _parse_timestamp(value)
+    if parsed is None:
+        return None
+    current = now or datetime.now(UTC)
+    return max(0, int((current - parsed).total_seconds()))
+
+
+def _latest_timestamp(*values: object) -> str | None:
+    latest_raw: str | None = None
+    latest_dt: datetime | None = None
+    for value in values:
+        normalized = _normalize_string(value)
+        parsed = _parse_timestamp(normalized)
+        if parsed is None:
+            continue
+        if latest_dt is None or parsed >= latest_dt:
+            latest_dt = parsed
+            latest_raw = normalized
+    return latest_raw
+
+
+def _compact_command(command: object, *, max_length: int = 140) -> str:
+    normalized = " ".join(str(command or "").split())
+    if len(normalized) <= max_length:
+        return normalized
+    return normalized[: max(0, max_length - 3)].rstrip() + "..."
+
+
 class BashExecService:
     def __init__(self, home: Path) -> None:
         self.home = home
@@ -224,8 +268,45 @@ class BashExecService:
         payload["status"] = _coerce_session_status(payload.get("status"))
         payload["last_progress"] = payload.get("last_progress") or read_json(self.progress_path(quest_root, str(payload.get("bash_id") or "")), None)
         payload["kind"] = str(payload.get("kind") or "exec")
+        payload = self._enrich_watchdog_fields(payload)
         return payload
 
+    @staticmethod
+    def _enrich_watchdog_fields(payload: dict[str, Any]) -> dict[str, Any]:
+        current = datetime.now(UTC)
+        last_progress = payload.get("last_progress")
+        last_progress_at = None
+        if isinstance(last_progress, dict):
+            last_progress_at = _normalize_string(last_progress.get("ts")) or None
+        last_output_at = _normalize_string(payload.get("last_output_at")) or None
+        latest_signal_at = _latest_timestamp(last_output_at, last_progress_at, payload.get("started_at"))
+        payload["last_progress_at"] = last_progress_at
+        payload["run_age_seconds"] = _age_seconds(payload.get("started_at"), now=current)
+        payload["status_age_seconds"] = _age_seconds(payload.get("updated_at"), now=current)
+        payload["silent_seconds"] = _age_seconds(last_output_at or payload.get("started_at"), now=current)
+        payload["progress_age_seconds"] = _age_seconds(last_progress_at, now=current)
+        payload["latest_signal_at"] = latest_signal_at
+        payload["signal_age_seconds"] = _age_seconds(latest_signal_at, now=current)
+        payload["watchdog_after_seconds"] = BASH_WATCHDOG_AFTER_SECONDS
+        payload["watchdog_overdue"] = (
+            payload.get("status") in {"running", "terminating"}
+            and isinstance(payload.get("signal_age_seconds"), int)
+            and int(payload["signal_age_seconds"]) >= BASH_WATCHDOG_AFTER_SECONDS
+        )
+        return payload
+
+    @staticmethod
+    def format_history_line(session: dict[str, Any]) -> str:
+        timestamp = (
+            _normalize_string(session.get("started_at"))
+            or _normalize_string(session.get("updated_at"))
+            or _normalize_string(session.get("finished_at"))
+            or "unknown-time"
+        )
+        command = _compact_command(session.get("command"))
+        bash_id = _normalize_string(session.get("bash_id") or session.get("id")) or "unknown-id"
+        return f"{timestamp} | {command} | {bash_id}"
+
     @staticmethod
     def _summary_session_payload(meta: dict[str, Any]) -> dict[str, Any]:
         return {
@@ -233,6 +314,7 @@ class BashExecService:
             "command": meta.get("command"),
             "kind": meta.get("kind") or "exec",
             "label": meta.get("label"),
+            "comment": meta.get("comment"),
             "workdir": meta.get("workdir"),
             "status": _coerce_session_status(meta.get("status")),
             "exit_code": meta.get("exit_code"),
@@ -241,6 +323,8 @@ class BashExecService:
             "finished_at": meta.get("finished_at"),
             "updated_at": meta.get("updated_at"),
             "last_progress": meta.get("last_progress"),
+            "last_output_at": meta.get("last_output_at"),
+            "last_output_seq": meta.get("last_output_seq"),
         }
 
     @staticmethod
@@ -469,6 +553,7 @@ class BashExecService:
         *,
         limit: int = DEFAULT_LOG_TAIL_LIMIT,
         before_seq: int | None = None,
+        after_seq: int | None = None,
         order: str = "asc",
     ) -> tuple[list[dict[str, Any]], dict[str, Any]]:
         if not self.meta_path(quest_root, bash_id).exists():
@@ -487,10 +572,13 @@ class BashExecService:
             else:
                 time.sleep(0.03)
             entries = read_jsonl(self.log_path(quest_root, bash_id))
+        latest_seq = int(entries[-1].get("seq") or 0) if entries else 0
         normalized_before = before_seq if isinstance(before_seq, int) and before_seq > 0 else None
+        normalized_after = after_seq if isinstance(after_seq, int) and after_seq >= 0 else None
+        if normalized_after is not None:
+            entries = [entry for entry in entries if int(entry.get("seq") or 0) > normalized_after]
         if normalized_before is not None:
             entries = [entry for entry in entries if int(entry.get("seq") or 0) < normalized_before]
-        latest_seq = int(entries[-1].get("seq") or 0) if entries else 0
         normalized_limit = max(1, limit)
         truncated = len(entries) > normalized_limit
         selected = entries[-normalized_limit:]
@@ -501,6 +589,8 @@ class BashExecService:
             "tail_limit": normalized_limit,
             "tail_start_seq": tail_start_seq if truncated else tail_start_seq,
             "latest_seq": latest_seq or None,
+            "after_seq": normalized_after,
+            "before_seq": normalized_before,
         }
         return selected, meta
 
@@ -521,7 +611,15 @@ class BashExecService:
                 return session
             time.sleep(max(0.1, poll_interval))
 
-    def request_stop(self, quest_root: Path, bash_id: str, *, reason: str | None = None, user_id: str | None = None) -> dict[str, Any]:
+    def request_stop(
+        self,
+        quest_root: Path,
+        bash_id: str,
+        *,
+        reason: str | None = None,
+        user_id: str | None = None,
+        force: bool = False,
+    ) -> dict[str, Any]:
         session = self.get_session(quest_root, bash_id)
         status = _normalize_string(session.get("status")).lower()
         if status in TERMINAL_STATUSES:
@@ -530,6 +628,7 @@ class BashExecService:
             "reason": _normalize_string(reason) or "user_stop",
             "user_id": _normalize_string(user_id) or _normalize_string(session.get("agent_id")) or "agent",
             "requested_at": utc_now(),
+            "force": bool(force),
         }
         _atomic_write_json(self.stop_request_path(quest_root, bash_id), request_payload)
         meta = read_json(self.meta_path(quest_root, bash_id), {})
@@ -540,12 +639,18 @@ class BashExecService:
         self._write_meta(quest_root, bash_id, meta)
         runtime = self._terminal_runtime_manager.get_runtime(quest_root, bash_id)
         if runtime is not None:
-            runtime.stop(reason=request_payload["reason"], force=False)
+            runtime.stop(reason=request_payload["reason"], force=bool(force))
         else:
             process_group_id = meta.get("process_group_id")
+            process_pid = meta.get("process_pid")
             if isinstance(process_group_id, int) and process_group_id > 0:
                 try:
-                    os.killpg(process_group_id, signal.SIGTERM)
+                    os.killpg(process_group_id, signal.SIGKILL if force else signal.SIGTERM)
+                except ProcessLookupError:
+                    pass
+            elif isinstance(process_pid, int) and process_pid > 0:
+                try:
+                    os.kill(process_pid, signal.SIGKILL if force else signal.SIGTERM)
                 except ProcessLookupError:
                     pass
         return self._session_payload(quest_root, meta)
@@ -561,6 +666,7 @@ class BashExecService:
         workdir_display: str,
         timeout_seconds: int | None,
         env_keys: list[str],
+        comment: str | dict[str, Any] | None = None,
         kind: str = "exec",
     ) -> dict[str, Any]:
         quest_root = context.require_quest_root().resolve()
@@ -582,6 +688,7 @@ class BashExecService:
             "agent_instance_id": agent_instance_id,
             "started_by_user_id": started_by_user_id,
             "stopped_by_user_id": None,
+            "comment": comment,
             "command": command,
             "workdir": workdir_display,
             "cwd": str(cwd),
@@ -592,6 +699,8 @@ class BashExecService:
             "exit_code": None,
             "stop_reason": None,
             "last_progress": None,
+            "last_output_at": None,
+            "last_output_seq": None,
             "started_at": timestamp,
             "finished_at": None,
             "updated_at": timestamp,
@@ -638,6 +747,7 @@ class BashExecService:
         workdir: str | None = None,
         env: dict[str, Any] | None = None,
         timeout_seconds: int | None = None,
+        comment: str | dict[str, Any] | None = None,
     ) -> dict[str, Any]:
         if not _normalize_string(command):
             raise ValueError("command_required")
@@ -656,6 +766,7 @@ class BashExecService:
             workdir_display=workdir_display,
             timeout_seconds=timeout_seconds,
             env_keys=sorted(env_payload),
+            comment=comment,
             kind="exec",
         )
         self.terminal_log_path(quest_root, bash_id).touch()
@@ -668,6 +779,7 @@ class BashExecService:
                 "bash_id": bash_id,
                 "quest_id": meta["quest_id"],
                 "command": command,
+                "comment": comment,
                 "workdir": workdir_display,
                 "mode": mode,
                 "started_at": meta["started_at"],
@@ -725,6 +837,8 @@ class BashExecService:
             "exit_code": None,
             "stop_reason": None,
             "last_progress": None,
+            "last_output_at": None,
+            "last_output_seq": None,
             "started_at": timestamp,
             "finished_at": None,
             "updated_at": timestamp,
@@ -1054,6 +1168,9 @@ class BashExecService:
             "bash_id": session["bash_id"],
             "log_path": session.get("log_path"),
             "status": session.get("status"),
+            "kind": session.get("kind"),
+            "comment": session.get("comment"),
+            "label": session.get("label"),
             "command": session.get("command"),
             "workdir": session.get("workdir"),
             "started_at": session.get("started_at"),
@@ -1061,6 +1178,17 @@ class BashExecService:
             "exit_code": session.get("exit_code"),
             "stop_reason": session.get("stop_reason"),
             "last_progress": session.get("last_progress"),
+            "last_progress_at": session.get("last_progress_at"),
+            "last_output_at": session.get("last_output_at"),
+            "last_output_seq": session.get("last_output_seq"),
+            "run_age_seconds": session.get("run_age_seconds"),
+            "status_age_seconds": session.get("status_age_seconds"),
+            "silent_seconds": session.get("silent_seconds"),
+            "progress_age_seconds": session.get("progress_age_seconds"),
+            "latest_signal_at": session.get("latest_signal_at"),
+            "signal_age_seconds": session.get("signal_age_seconds"),
+            "watchdog_after_seconds": session.get("watchdog_after_seconds"),
+            "watchdog_overdue": session.get("watchdog_overdue"),
         }
         if include_log:
             result["log"] = self.read_terminal_log(quest_root, str(session["bash_id"]))

package/src/deepscientist/bridges/base.py

@@ -10,6 +10,8 @@ from urllib.error import URLError
 from urllib.parse import parse_qs
 from urllib.request import Request, urlopen
 
+from ..connector_runtime import parse_conversation_id
+
 
 @dataclass
 class BridgeWebhookResult:
@@ -52,19 +54,6 @@ class BaseConnectorBridge:
         }
 
     def deliver(self, payload: dict[str, Any], config: dict[str, Any]) -> dict[str, Any] | None:
-        relay_url = str(config.get("relay_url") or "").strip()
-        if relay_url:
-            envelope = {
-                "bridge_version": "deepscientist-connector-bridge/v1",
-                "connector": self.name,
-                "payload": self.format_outbound(payload, config),
-                "normalized_payload": payload,
-            }
-            headers = {"Content-Type": "application/json; charset=utf-8"}
-            token = str(config.get("relay_auth_token") or "").strip()
-            if token:
-                headers["Authorization"] = f"Bearer {token}"
-            return self._post_json(relay_url, envelope, headers=headers)
         return self.deliver_direct(payload, config)
 
     def deliver_direct(self, payload: dict[str, Any], config: dict[str, Any]) -> dict[str, Any] | None:
@@ -72,18 +61,20 @@ class BaseConnectorBridge:
 
     @staticmethod
     def extract_target(conversation_id: Any) -> dict[str, str]:
-        raw = str(conversation_id or "").strip()
-        parts = raw.split(":", 2)
-        if len(parts) == 3:
+        parsed = parse_conversation_id(conversation_id)
+        if parsed is not None:
             return {
-                "connector": parts[0],
-                "chat_type": parts[1],
-                "chat_id": parts[2],
+                "connector": str(parsed.get("connector") or ""),
+                "chat_type": str(parsed.get("chat_type") or ""),
+                "chat_id": str(parsed.get("chat_id") or ""),
+                "profile_id": str(parsed.get("profile_id") or ""),
             }
+        raw = str(conversation_id or "").strip()
         return {
             "connector": "",
             "chat_type": "",
             "chat_id": raw,
+            "profile_id": "",
         }
 
     @staticmethod

package/src/deepscientist/channels/discord_gateway.py

@@ -10,6 +10,7 @@ from urllib.request import Request, urlopen
 from websockets.exceptions import ConnectionClosed
 from websockets.sync.client import connect as websocket_connect
 
+from ..connector_runtime import format_conversation_id
 from ..shared import read_json, utc_now, write_json
 
 
@@ -24,11 +25,17 @@ class DiscordGatewayService:
         config: dict[str, Any],
         on_event: Callable[[dict[str, Any]], None],
         log: Callable[[str, str], None] | None = None,
+        profile_id: str | None = None,
+        profile_label: str | None = None,
+        encode_profile_id: bool = False,
     ) -> None:
         self.home = home
         self.config = config
         self.on_event = on_event
         self.log = log or self._default_log
+        self.profile_id = str(profile_id or "").strip() or None
+        self.profile_label = str(profile_label or "").strip() or None
+        self._encode_profile_id = bool(encode_profile_id and self.profile_id)
         self._thread: threading.Thread | None = None
         self._stop_event = threading.Event()
         self._heartbeat_stop = threading.Event()
@@ -37,6 +44,8 @@ class DiscordGatewayService:
         self._seq: int | None = None
         self._bot_user_id: str | None = None
         self._root = home / "logs" / "connectors" / "discord"
+        if self.profile_id:
+            self._root = self._root / "profiles" / self.profile_id
         self._runtime_path = self._root / "runtime.json"
 
     def start(self) -> bool:
@@ -71,7 +80,7 @@ class DiscordGatewayService:
         self._thread = threading.Thread(
             target=self._run,
             daemon=True,
-            name="deepscientist-discord-gateway",
+            name=f"deepscientist-discord-gateway-{self.profile_id or 'default'}",
         )
         self._thread.start()
         return True
@@ -288,12 +297,22 @@ class DiscordGatewayService:
             "sender_id": sender_id,
             "sender_name": str(author.get("global_name") or author.get("username") or sender_id).strip(),
             "message_id": str(data.get("id") or "").strip(),
-            "conversation_id": f"discord:{chat_type}:{channel_id}",
+            "conversation_id": self._conversation_id(chat_type, channel_id),
+            "profile_id": self.profile_id,
+            "profile_label": self.profile_label,
             "text": normalized_text,
             "mentioned": mentioned,
             "raw_event": data,
         }
 
+    def _conversation_id(self, chat_type: str, chat_id: str) -> str:
+        return format_conversation_id(
+            "discord",
+            chat_type,
+            chat_id,
+            profile_id=self.profile_id if self._encode_profile_id else None,
+        )
+
     @staticmethod
     def _strip_bot_mention(text: str, bot_user_id: str) -> str:
         cleaned = str(text or "").strip()
@@ -354,6 +373,10 @@ class DiscordGatewayService:
         state = read_json(self._runtime_path, {}) or {}
         if not isinstance(state, dict):
             state = {}
+        if self.profile_id:
+            state["profile_id"] = self.profile_id
+        if self.profile_label:
+            state["profile_label"] = self.profile_label
         state.update(patch)
         write_json(self._runtime_path, state)
 

package/src/deepscientist/channels/feishu_long_connection.py

@@ -9,6 +9,7 @@ from pathlib import Path
 from typing import Any, Callable
 
 from ..bridges.connectors import FeishuConnectorBridge
+from ..connector_runtime import format_conversation_id, parse_conversation_id
 from ..shared import read_json, utc_now, write_json
 
 
@@ -29,17 +30,25 @@ class FeishuLongConnectionService:
         config: dict[str, Any],
         on_event: Callable[[dict[str, Any]], None],
         log: Callable[[str, str], None] | None = None,
+        profile_id: str | None = None,
+        profile_label: str | None = None,
+        encode_profile_id: bool = False,
     ) -> None:
         self.home = home
         self.config = config
         self.on_event = on_event
         self.log = log or self._default_log
+        self.profile_id = str(profile_id or "").strip() or None
+        self.profile_label = str(profile_label or "").strip() or None
+        self._encode_profile_id = bool(encode_profile_id and self.profile_id)
         self._thread: threading.Thread | None = None
         self._stop_event = threading.Event()
         self._loop: asyncio.AbstractEventLoop | None = None
         self._async_stop: asyncio.Event | None = None
         self._client: Any = None
         self._root = home / "logs" / "connectors" / "feishu"
+        if self.profile_id:
+            self._root = self._root / "profiles" / self.profile_id
         self._runtime_path = self._root / "runtime.json"
 
     def start(self) -> bool:
@@ -85,7 +94,7 @@ class FeishuLongConnectionService:
         self._thread = threading.Thread(
             target=self._run,
             daemon=True,
-            name="deepscientist-feishu-long-connection",
+            name=f"deepscientist-feishu-long-connection-{self.profile_id or 'default'}",
         )
         self._thread.start()
         return True
@@ -201,16 +210,41 @@ class FeishuLongConnectionService:
             config=self.config,
         )
         for event in result.events:
-            self.on_event(event)
+            normalized = self._normalize_event(event)
+            self.on_event(normalized)
             self._write_state(
                 connected=True,
                 connection_state="connected",
                 auth_state="ready",
                 last_event_at=utc_now(),
-                last_conversation_id=event.get("conversation_id"),
+                last_conversation_id=normalized.get("conversation_id"),
                 updated_at=utc_now(),
             )
 
+    def _normalize_event(self, event: dict[str, Any]) -> dict[str, Any]:
+        normalized = dict(event)
+        chat_type = str(normalized.get("chat_type") or "direct").strip().lower() or "direct"
+        group_id = str(normalized.get("group_id") or "").strip()
+        direct_id = str(normalized.get("direct_id") or "").strip()
+        chat_id = group_id if chat_type == "group" else direct_id
+        if not chat_id:
+            parsed = parse_conversation_id(normalized.get("conversation_id"))
+            if parsed is not None:
+                chat_type = str(parsed.get("chat_type") or chat_type).strip().lower() or chat_type
+                chat_id = str(parsed.get("chat_id") or "").strip()
+        normalized["conversation_id"] = self._conversation_id(chat_type, chat_id or "unknown")
+        normalized["profile_id"] = self.profile_id
+        normalized["profile_label"] = self.profile_label
+        return normalized
+
+    def _conversation_id(self, chat_type: str, chat_id: str) -> str:
+        return format_conversation_id(
+            "feishu",
+            chat_type,
+            chat_id,
+            profile_id=self.profile_id if self._encode_profile_id else None,
+        )
+
     @staticmethod
     def _sdk_bundle() -> dict[str, Any] | None:
         try:
@@ -239,6 +273,10 @@ class FeishuLongConnectionService:
         state = read_json(self._runtime_path, {}) or {}
         if not isinstance(state, dict):
             state = {}
+        if self.profile_id:
+            state["profile_id"] = self.profile_id
+        if self.profile_label:
+            state["profile_label"] = self.profile_label
         state.update(patch)
         write_json(self._runtime_path, state)