@researai/deepscientist 1.5.11 → 1.5.12

package/src/deepscientist/bash_exec/service.py

@@ -11,12 +11,13 @@ import sys
 import tempfile
 import threading
 import time
+from collections import deque
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
 
 from ..mcp.context import McpContext
-from ..shared import append_jsonl, ensure_dir, generate_id, read_json, read_jsonl, utc_now
+from ..shared import append_jsonl, ensure_dir, generate_id, iter_jsonl, read_json, read_jsonl, read_jsonl_tail, utc_now
 from .runtime import TerminalRuntimeManager
 
 BASH_STATUS_MARKER_PREFIX = "__DS_BASH_STATUS__"
@@ -24,6 +25,9 @@ BASH_CARRIAGE_RETURN_PREFIX = "__DS_BASH_CR__"
 BASH_PROGRESS_PREFIX = "__DS_PROGRESS__"
 BASH_TERMINAL_PROMPT_PREFIX = "__DS_TERMINAL_PROMPT__"
 DEFAULT_LOG_TAIL_LIMIT = 200
+DEFAULT_INLINE_BASH_LOG_LINE_LIMIT = 2000
+DEFAULT_INLINE_BASH_LOG_HEAD_LINES = 500
+DEFAULT_INLINE_BASH_LOG_TAIL_LINES = 1500
 DEFAULT_POLL_INTERVAL_SECONDS = 0.35
 TERMINAL_STATUSES = {"completed", "failed", "terminated"}
 DEFAULT_TERMINAL_SESSION_ID = "terminal-main"
@@ -46,6 +50,52 @@ def _atomic_write_json(path: Path, payload: Any) -> None:
     temp_path.replace(path)
 
 
+def _count_jsonl_records(path: Path) -> int:
+    return sum(1 for _ in iter_jsonl(path))
+
+
+def _build_terminal_log_preview_payload(path: Path) -> dict[str, Any]:
+    if not path.exists():
+        return {
+            "log": "",
+            "log_line_count": 0,
+            "log_truncated": False,
+        }
+
+    head_lines: list[str] = []
+    tail_lines: deque[str] = deque(maxlen=DEFAULT_INLINE_BASH_LOG_TAIL_LINES)
+    total = 0
+    with path.open("r", encoding="utf-8", errors="replace") as handle:
+        for raw_line in handle:
+            line = raw_line.rstrip("\n")
+            total += 1
+            if total <= DEFAULT_INLINE_BASH_LOG_HEAD_LINES:
+                head_lines.append(line)
+            tail_lines.append(line)
+
+    if total <= DEFAULT_INLINE_BASH_LOG_LINE_LIMIT:
+        return {
+            "log": "\n".join(list(tail_lines)),
+            "log_line_count": total,
+            "log_truncated": False,
+        }
+
+    omitted = max(0, total - DEFAULT_INLINE_BASH_LOG_HEAD_LINES - DEFAULT_INLINE_BASH_LOG_TAIL_LINES)
+    marker = (
+        "[... omitted "
+        f"{omitted} lines from the middle of this log. "
+        "Use bash_exec(mode='read', id=..., start=..., tail=...) for a specific window.]"
+    )
+    return {
+        "log": "\n".join(head_lines + [marker] + list(tail_lines)),
+        "log_line_count": total,
+        "log_truncated": True,
+        "log_preview_head_lines": DEFAULT_INLINE_BASH_LOG_HEAD_LINES,
+        "log_preview_tail_lines": DEFAULT_INLINE_BASH_LOG_TAIL_LINES,
+        "log_preview_omitted_lines": omitted,
+    }
+
+
 def _normalize_string(value: object) -> str:
     return str(value or "").strip()
 
@@ -568,7 +618,8 @@ class BashExecService:
         if not self.meta_path(quest_root, bash_id).exists():
             raise FileNotFoundError(f"Unknown bash session `{bash_id}`.")
         deadline = time.monotonic() + 0.6
-        entries = read_jsonl(self.log_path(quest_root, bash_id))
+        path = self.log_path(quest_root, bash_id)
+        entries = read_jsonl_tail(path, max(1, limit))
         while time.monotonic() < deadline:
             if any(str(entry.get("stream") or "") not in {"system", "prompt"} for entry in entries):
                 break
@@ -580,24 +631,33 @@ class BashExecService:
                 time.sleep(0.05)
             else:
                 time.sleep(0.03)
-            entries = read_jsonl(self.log_path(quest_root, bash_id))
+            entries = read_jsonl_tail(path, max(1, limit))
         latest_seq = int(entries[-1].get("seq") or 0) if entries else 0
         normalized_before = before_seq if isinstance(before_seq, int) and before_seq > 0 else None
         normalized_after = after_seq if isinstance(after_seq, int) and after_seq >= 0 else None
-        if normalized_after is not None:
-            entries = [entry for entry in entries if int(entry.get("seq") or 0) > normalized_after]
-        if normalized_before is not None:
-            entries = [entry for entry in entries if int(entry.get("seq") or 0) < normalized_before]
-        selection_pool = entries
-        if prefer_visible:
-            visible_entries = [
-                entry for entry in entries if str(entry.get("stream") or "") not in {"system", "prompt"}
-            ]
-            if visible_entries:
-                selection_pool = visible_entries
         normalized_limit = max(1, limit)
-        truncated = len(selection_pool) > normalized_limit
-        selected = selection_pool[-normalized_limit:]
+        selection_pool: deque[dict[str, Any]] = deque(maxlen=normalized_limit)
+        visible_pool: deque[dict[str, Any]] = deque(maxlen=normalized_limit)
+        total_filtered = 0
+        for entry in iter_jsonl(path):
+            seq = int(entry.get("seq") or 0)
+            latest_seq = max(latest_seq, seq)
+            if normalized_after is not None and seq <= normalized_after:
+                continue
+            if normalized_before is not None and seq >= normalized_before:
+                continue
+            total_filtered += 1
+            selection_pool.append(entry)
+            if str(entry.get("stream") or "") not in {"system", "prompt"}:
+                visible_pool.append(entry)
+        selected_source: list[dict[str, Any]]
+        if prefer_visible and visible_pool:
+            selected_source = list(visible_pool)
+            truncated = total_filtered > len(visible_pool)
+        else:
+            selected_source = list(selection_pool)
+            truncated = total_filtered > len(selection_pool)
+        selected = selected_source[-normalized_limit:]
         if order == "desc":
             selected = list(reversed(selected))
         tail_start_seq = int(selected[0].get("seq") or 0) if selected else None
@@ -868,7 +928,7 @@ class BashExecService:
             "last_input_at": None,
             "last_prompt_at": None,
             "last_command": None,
-            "history_count": len(read_jsonl(self.history_path(quest_root, bash_id))),
+            "history_count": _count_jsonl_records(self.history_path(quest_root, bash_id)),
         }
 
     def ensure_terminal_session(
@@ -918,7 +978,7 @@ class BashExecService:
         self.prompt_events_path(resolved_quest_root, bash_id).touch()
         _atomic_write_json(
             self.input_cursor_path(resolved_quest_root, bash_id),
-            {"offset": len(read_jsonl(self.input_path(resolved_quest_root, bash_id))), "updated_at": utc_now()},
+            {"offset": _count_jsonl_records(self.input_path(resolved_quest_root, bash_id)), "updated_at": utc_now()},
         )
         _atomic_write_json(
             self.line_buffer_path(resolved_quest_root, bash_id),
@@ -1072,7 +1132,7 @@ class BashExecService:
                 append_jsonl(self.history_path(quest_root, bash_id), item)
             meta = read_json(self.meta_path(quest_root, bash_id), {})
             meta["last_command"] = completed[-1]["command"]
-            meta["history_count"] = len(read_jsonl(self.history_path(quest_root, bash_id)))
+            meta["history_count"] = _count_jsonl_records(self.history_path(quest_root, bash_id))
             meta["updated_at"] = utc_now()
             meta["last_input_at"] = utc_now()
             self._write_meta(quest_root, bash_id, meta)
@@ -1138,7 +1198,7 @@ class BashExecService:
             before_seq=None,
             order="asc",
         )
-        history = read_jsonl(self.history_path(quest_root, bash_id))
+        history = read_jsonl_tail(self.history_path(quest_root, bash_id), max(1, command_limit))
         latest_commands = [
             {
                 "command_id": item.get("command_id"),
@@ -1208,7 +1268,7 @@ class BashExecService:
             "watchdog_overdue": session.get("watchdog_overdue"),
         }
         if include_log:
-            result["log"] = self.read_terminal_log(quest_root, str(session["bash_id"]))
+            result.update(self._log_preview_payload(quest_root, str(session["bash_id"])))
         if export_log or _normalize_string(export_log_to):
             cwd, _ = self.resolve_workdir(context, str(session.get("workdir") or ""))
             result.update(
@@ -1221,3 +1281,6 @@ class BashExecService:
                 )
             )
         return result
+
+    def _log_preview_payload(self, quest_root: Path, bash_id: str) -> dict[str, Any]:
+        return _build_terminal_log_preview_payload(self.terminal_log_path(quest_root, bash_id))

package/src/deepscientist/channels/local.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 from pathlib import Path
 from typing import Any
 
-from ..shared import append_jsonl, ensure_dir, read_jsonl, utc_now
+from ..shared import append_jsonl, count_jsonl, ensure_dir, read_jsonl, utc_now
 from .base import BaseChannel
 
 
@@ -27,6 +27,6 @@ class LocalChannel(BaseChannel):
         return {
             "name": self.name,
             "display_mode": self.display_mode,
-            "inbox_count": len(read_jsonl(self.root / "inbox.jsonl")),
-            "outbox_count": len(read_jsonl(self.root / "outbox.jsonl")),
+            "inbox_count": count_jsonl(self.root / "inbox.jsonl"),
+            "outbox_count": count_jsonl(self.root / "outbox.jsonl"),
         }

package/src/deepscientist/channels/qq.py

@@ -6,7 +6,7 @@ from typing import Any
 from ..connector_runtime import build_discovered_target, conversation_identity_key, format_conversation_id, merge_discovered_targets, parse_conversation_id
 from ..bridges import get_connector_bridge
 from ..connector.qq_profiles import find_qq_profile, list_qq_profiles, merge_qq_profile_config, qq_profile_label
-from ..shared import append_jsonl, ensure_dir, generate_id, read_json, read_jsonl, utc_now, write_json
+from ..shared import append_jsonl, count_jsonl, ensure_dir, generate_id, read_json, read_jsonl, read_jsonl_tail, utc_now, write_json
 from .base import BaseChannel
 
 
@@ -387,9 +387,9 @@ class QQRelayChannel(BaseChannel):
             "main_chat_id": main_chat_id,
             "last_conversation_id": last_conversation_id,
             "last_error": last_error,
-            "inbox_count": len(read_jsonl(self.inbox_path)),
-            "outbox_count": len(read_jsonl(self.outbox_path)),
-            "ignored_count": len(read_jsonl(self.ignored_path)),
+            "inbox_count": count_jsonl(self.inbox_path),
+            "outbox_count": count_jsonl(self.outbox_path),
+            "ignored_count": count_jsonl(self.ignored_path),
             "binding_count": len(bindings),
             "bindings": bindings,
             "known_targets": known_targets,
@@ -947,15 +947,15 @@ class QQRelayChannel(BaseChannel):
 
     def _recent_events(self) -> list[dict[str, Any]]:
         events: list[dict[str, Any]] = []
-        for record in read_jsonl(self.inbox_path)[-self.recent_event_limit :]:
+        for record in read_jsonl_tail(self.inbox_path, self.recent_event_limit):
             event = self._build_recent_event("inbound", record)
             if event is not None:
                 events.append(event)
-        for record in read_jsonl(self.outbox_path)[-self.recent_event_limit :]:
+        for record in read_jsonl_tail(self.outbox_path, self.recent_event_limit):
             event = self._build_recent_event("outbound", record)
             if event is not None:
                 events.append(event)
-        for record in read_jsonl(self.ignored_path)[-self.recent_event_limit :]:
+        for record in read_jsonl_tail(self.ignored_path, self.recent_event_limit):
             event = self._build_recent_event("ignored", record)
             if event is not None:
                 events.append(event)

package/src/deepscientist/channels/relay.py

@@ -21,7 +21,7 @@ from ..connector.connector_profiles import (
     merge_connector_profile_config,
 )
 from ..bridges import get_connector_bridge
-from ..shared import append_jsonl, ensure_dir, generate_id, read_json, read_jsonl, utc_now, write_json
+from ..shared import append_jsonl, count_jsonl, ensure_dir, generate_id, read_json, read_jsonl, read_jsonl_tail, utc_now, write_json
 from .base import BaseChannel
 
 
@@ -412,9 +412,9 @@ class GenericRelayChannel(BaseChannel):
                 ),
                 str(runtime_state.get("last_error") or "").strip() or None if isinstance(runtime_state, dict) else None,
             ),
-            "inbox_count": len(read_jsonl(self.inbox_path)),
-            "outbox_count": len(read_jsonl(self.outbox_path)),
-            "ignored_count": len(read_jsonl(self.ignored_path)),
+            "inbox_count": count_jsonl(self.inbox_path),
+            "outbox_count": count_jsonl(self.outbox_path),
+            "ignored_count": count_jsonl(self.ignored_path),
             "binding_count": len(bindings),
             "bindings": bindings,
             "known_targets": known_targets,
@@ -894,15 +894,15 @@ class GenericRelayChannel(BaseChannel):
 
     def _recent_events(self) -> list[dict[str, Any]]:
         events: list[dict[str, Any]] = []
-        for record in read_jsonl(self.inbox_path)[-self.recent_event_limit :]:
+        for record in read_jsonl_tail(self.inbox_path, self.recent_event_limit):
             event = self._build_recent_event("inbound", record)
             if event is not None:
                 events.append(event)
-        for record in read_jsonl(self.outbox_path)[-self.recent_event_limit :]:
+        for record in read_jsonl_tail(self.outbox_path, self.recent_event_limit):
             event = self._build_recent_event("outbound", record)
             if event is not None:
                 events.append(event)
-        for record in read_jsonl(self.ignored_path)[-self.recent_event_limit :]:
+        for record in read_jsonl_tail(self.ignored_path, self.recent_event_limit):
             event = self._build_recent_event("ignored", record)
             if event is not None:
                 events.append(event)

package/src/deepscientist/channels/weixin_ilink.py

@@ -19,7 +19,10 @@ from ..connector.weixin_support import (
     save_weixin_get_updates_buf,
 )
 
-_SESSION_PAUSE_SECONDS = 60 * 60
+_SESSION_RETRY_INITIAL_SECONDS = 5.0
+_SESSION_RETRY_MAX_SECONDS = 60.0
+_POLL_RETRY_INITIAL_SECONDS = 2.0
+_POLL_RETRY_MAX_SECONDS = 30.0
 
 
 class WeixinIlinkService:
@@ -89,7 +92,8 @@ class WeixinIlinkService:
 
     def _run(self) -> None:
         timeout_ms = DEFAULT_WEIXIN_LONG_POLL_TIMEOUT_MS
-        pause_until = 0.0
+        retry_until = 0.0
+        retry_reason: str | None = None
         sync_buf = load_weixin_get_updates_buf(self._root)
         base_url = normalize_weixin_base_url(self.config.get("base_url"))
         cdn_base_url = normalize_weixin_cdn_base_url(self.config.get("cdn_base_url"))
@@ -97,6 +101,10 @@ class WeixinIlinkService:
         login_user_id = str(self.config.get("login_user_id") or "").strip() or None
         token = self._secret("bot_token", "bot_token_env")
         route_tag = str(self.config.get("route_tag") or "").strip() or None
+        session_retry_seconds = _SESSION_RETRY_INITIAL_SECONDS
+        poll_retry_seconds = _POLL_RETRY_INITIAL_SECONDS
+        session_expired_count = 0
+        session_expired_since: str | None = None
 
         self._write_state(
             enabled=True,
@@ -108,20 +116,31 @@ class WeixinIlinkService:
             login_user_id=login_user_id,
             base_url=base_url,
             cdn_base_url=cdn_base_url,
+            retry_reason=None,
+            retry_after_seconds=None,
+            pause_until=None,
             updated_at=utc_now(),
         )
 
         while not self._stop_event.is_set():
             now = time.time()
-            if pause_until > now:
-                self._write_state(
-                    connected=False,
-                    connection_state="paused",
-                    auth_state="error",
-                    pause_until=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(pause_until)),
-                    updated_at=utc_now(),
-                )
-                self._stop_event.wait(min(max(pause_until - now, 1.0), 5.0))
+            if retry_until > now:
+                retry_after_seconds = max(int(retry_until - now + 0.999), 1)
+                state_patch: dict[str, Any] = {
+                    "connected": False,
+                    "connection_state": "connecting" if retry_reason == "session_expired" else "error",
+                    "auth_state": "ready" if token and account_id else "missing_credentials",
+                    "retry_reason": retry_reason,
+                    "retry_after_seconds": retry_after_seconds,
+                    "session_expired_count": session_expired_count or None,
+                    "session_expired_since": session_expired_since,
+                    "pause_until": None,
+                    "updated_at": utc_now(),
+                }
+                if retry_reason == "session_expired":
+                    state_patch["last_error"] = f"session expired ({SESSION_EXPIRED_ERRCODE}); retrying automatically"
+                self._write_state(**state_patch)
+                self._stop_event.wait(min(max(retry_until - now, 0.5), 5.0))
                 continue
             try:
                 response = get_weixin_updates(
@@ -137,14 +156,35 @@ class WeixinIlinkService:
                 errcode = int(response.get("errcode") or 0)
                 retcode = int(response.get("ret") or 0)
                 if errcode == SESSION_EXPIRED_ERRCODE or retcode == SESSION_EXPIRED_ERRCODE:
-                    pause_until = time.time() + _SESSION_PAUSE_SECONDS
-                    self.log("warning", "weixin.ilink: session expired; pausing for one hour")
+                    session_expired_count += 1
+                    if session_expired_since is None:
+                        session_expired_since = utc_now()
+                    if sync_buf:
+                        sync_buf = ""
+                        save_weixin_get_updates_buf(self._root, "")
+                    retry_delay_seconds = session_retry_seconds
+                    retry_after_seconds = max(int(retry_delay_seconds + 0.999), 1)
+                    session_retry_seconds = min(session_retry_seconds * 2.0, _SESSION_RETRY_MAX_SECONDS)
+                    retry_reason = "session_expired"
+                    retry_until = time.time() + retry_delay_seconds
+                    timeout_ms = DEFAULT_WEIXIN_LONG_POLL_TIMEOUT_MS
+                    self.log(
+                        "warning",
+                        (
+                            "weixin.ilink: session expired; cleared sync state and "
+                            f"retrying in {retry_after_seconds}s"
+                        ),
+                    )
                     self._write_state(
                         connected=False,
-                        connection_state="paused",
-                        auth_state="error",
-                        last_error=f"session expired ({SESSION_EXPIRED_ERRCODE})",
-                        pause_until=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(pause_until)),
+                        connection_state="connecting",
+                        auth_state="ready" if token and account_id else "missing_credentials",
+                        last_error=f"session expired ({SESSION_EXPIRED_ERRCODE}); retrying automatically",
+                        retry_reason=retry_reason,
+                        retry_after_seconds=retry_after_seconds,
+                        session_expired_count=session_expired_count,
+                        session_expired_since=session_expired_since,
+                        pause_until=None,
                         updated_at=utc_now(),
                     )
                     continue
@@ -156,11 +196,26 @@ class WeixinIlinkService:
                 if next_sync_buf:
                     sync_buf = next_sync_buf
                     save_weixin_get_updates_buf(self._root, sync_buf)
+                if session_expired_count > 0:
+                    self.log(
+                        "info",
+                        f"weixin.ilink: session recovered after {session_expired_count} retry attempt(s)",
+                    )
+                retry_reason = None
+                retry_until = 0.0
+                session_retry_seconds = _SESSION_RETRY_INITIAL_SECONDS
+                poll_retry_seconds = _POLL_RETRY_INITIAL_SECONDS
+                session_expired_count = 0
+                session_expired_since = None
                 self._write_state(
                     connected=True,
                     connection_state="connected",
                     auth_state="ready",
                     last_error=None,
+                    retry_reason=None,
+                    retry_after_seconds=None,
+                    session_expired_count=None,
+                    session_expired_since=None,
                     pause_until=None,
                     updated_at=utc_now(),
                 )
@@ -182,18 +237,34 @@ class WeixinIlinkService:
             except Exception as exc:
                 if self._stop_event.is_set():
                     break
-                self.log("warning", f"weixin.ilink: polling failed: {exc}")
+                retry_reason = "poll_error"
+                retry_delay_seconds = poll_retry_seconds
+                retry_after_seconds = max(int(retry_delay_seconds + 0.999), 1)
+                retry_until = time.time() + retry_delay_seconds
+                poll_retry_seconds = min(poll_retry_seconds * 2.0, _POLL_RETRY_MAX_SECONDS)
+                self.log(
+                    "warning",
+                    f"weixin.ilink: polling failed: {exc}; retrying in {retry_after_seconds}s",
+                )
                 self._write_state(
                     connected=False,
                     connection_state="error",
                     auth_state="ready" if token and account_id else "missing_credentials",
                     last_error=str(exc),
+                    retry_reason=retry_reason,
+                    retry_after_seconds=retry_after_seconds,
+                    session_expired_count=session_expired_count or None,
+                    session_expired_since=session_expired_since,
+                    pause_until=None,
                     updated_at=utc_now(),
                 )
-                self._stop_event.wait(2.0)
+                self._stop_event.wait(retry_delay_seconds)
         self._write_state(
             connected=False,
             connection_state="stopped",
+            retry_reason=None,
+            retry_after_seconds=None,
+            pause_until=None,
             updated_at=utc_now(),
         )
 

package/src/deepscientist/config/models.py

@@ -95,6 +95,7 @@ def default_runners() -> dict:
             "enabled": True,
             "binary": "codex",
             "config_dir": "~/.codex",
+            "profile": "",
             "model": "gpt-5.4",
             "model_reasoning_effort": "xhigh",
             "approval_policy": "on-request",