@researai/deepscientist 1.5.16 → 1.5.17

package/src/deepscientist/daemon/app.py

@@ -55,6 +55,7 @@ from ..connector.connector_profiles import (
 from ..connector_runtime import conversation_identity_key, format_conversation_id, normalize_conversation_id, parse_conversation_id
 from ..config import ConfigManager
 from ..config.models import SYSTEM_CONNECTOR_NAMES
+from ..diagnostics import FailureDiagnosis, diagnose_runner_failure
 from ..home import repo_root
 from ..memory import MemoryService
 from ..network import urlopen_with_proxy as urlopen
@@ -105,6 +106,8 @@ TERMINAL_STREAM_IDLE_SLEEP_SECONDS = 0.02
 _AUTO_CONTINUE_DELAY_SECONDS = 240.0
 _AUTO_CONTINUE_ACTIVE_WORK_DELAY_SECONDS = 0.2
 _TERMINAL_PREWARM_DEBOUNCE_SECONDS = 20.0
+_STALLED_RUNNING_TURN_INACTIVITY_SECONDS = 30 * 60
+_STALLED_RUNNING_TURN_INTERRUPT_TIMEOUT_SECONDS = 5.0
 CODEX_RETRY_DEFAULT_MAX_ATTEMPTS = 5
 CODEX_RETRY_DEFAULT_INITIAL_BACKOFF_SEC = 10.0
 CODEX_RETRY_DEFAULT_BACKOFF_MULTIPLIER = 6.0
@@ -755,7 +758,23 @@ class DaemonApp:
                     if int(snapshot.get("pending_user_message_count") or 0) > 0
                     else "auto_continue"
                 )
-                scheduled = self.schedule_turn(quest_id, reason=reason)
+                retry_delay_seconds = self._recovery_retry_delay_seconds(snapshot) if reason == "auto_continue" else None
+                if retry_delay_seconds is not None and retry_delay_seconds > 0:
+                    self._schedule_turn_later(
+                        quest_id,
+                        reason=reason,
+                        delay_seconds=retry_delay_seconds,
+                    )
+                    scheduled = {
+                        "scheduled": True,
+                        "started": False,
+                        "queued": True,
+                        "reason": reason,
+                        "delayed": True,
+                        "delay_seconds": retry_delay_seconds,
+                    }
+                else:
+                    scheduled = self.schedule_turn(quest_id, reason=reason)
                 event = {
                     "event_id": generate_id("evt"),
                     "type": "quest.runtime_auto_resumed",
@@ -767,6 +786,8 @@ class DaemonApp:
                     "scheduled": bool(scheduled.get("scheduled")),
                     "started": bool(scheduled.get("started")),
                     "queued": bool(scheduled.get("queued")),
+                    "delayed": bool(scheduled.get("delayed")),
+                    "delay_seconds": scheduled.get("delay_seconds"),
                     "created_at": utc_now(),
                 }
                 append_jsonl(self.home / "quests" / quest_id / ".ds" / "events.jsonl", event)
@@ -781,6 +802,8 @@ class DaemonApp:
                     scheduled=bool(scheduled.get("scheduled")),
                     started=bool(scheduled.get("started")),
                     queued=bool(scheduled.get("queued")),
+                    delayed=bool(scheduled.get("delayed")),
+                    delay_seconds=scheduled.get("delay_seconds"),
                 )
                 self._recovered_quest_ids.add(quest_id)
                 resumed.append(
@@ -834,6 +857,63 @@ class DaemonApp:
             count += 1
         return count
 
+    def _recovery_retry_delay_seconds(self, snapshot: dict[str, Any]) -> float | None:
+        retry_state = snapshot.get("retry_state") if isinstance(snapshot.get("retry_state"), dict) else None
+        if not retry_state:
+            return None
+        next_retry_at = str(retry_state.get("next_retry_at") or "").strip()
+        if not next_retry_at:
+            return None
+        parsed = self._parse_event_timestamp(next_retry_at)
+        if parsed is None:
+            return None
+        return max((parsed - datetime.now(UTC)).total_seconds(), 0.0)
+
+    def _resume_retry_state(
+        self,
+        snapshot: dict[str, Any],
+        *,
+        max_attempts: int,
+    ) -> tuple[int, str | None, dict[str, Any] | None]:
+        retry_state = snapshot.get("retry_state") if isinstance(snapshot.get("retry_state"), dict) else None
+        resume_source = str(snapshot.get("last_resume_source") or "").strip()
+        if not retry_state or not resume_source.startswith("auto:daemon-recovery"):
+            return 1, None, None
+
+        try:
+            recorded_attempt = int(retry_state.get("attempt_index") or 0)
+        except (TypeError, ValueError):
+            recorded_attempt = 0
+        if recorded_attempt <= 0:
+            return 1, None, None
+
+        next_retry_at = str(retry_state.get("next_retry_at") or "").strip()
+        start_attempt = recorded_attempt + 1 if next_retry_at else recorded_attempt
+        if start_attempt > max_attempts:
+            start_attempt = max_attempts
+        if start_attempt <= 1:
+            return 1, None, None
+
+        turn_id = str(retry_state.get("turn_id") or "").strip() or None
+        previous_run_id = str(retry_state.get("last_run_id") or "").strip() or None
+        failure_summary = str(retry_state.get("last_error") or "").strip() or None
+        retry_context = {
+            "turn_id": turn_id,
+            "attempt_index": recorded_attempt,
+            "max_attempts": max_attempts,
+            "previous_run_id": previous_run_id,
+            "failure_kind": "daemon_recovery",
+            "failure_summary": failure_summary or "Recovered retry state after daemon restart.",
+            "previous_exit_code": None,
+            "previous_output_text": "",
+            "stderr_tail": "",
+            "recent_messages": [],
+            "tool_progress": [],
+            "workspace_summary": {},
+            "recent_artifacts": [],
+        }
+        return start_attempt, turn_id, retry_context
+
     def _record_auto_resume_suppressed(
         self,
         *,
@@ -1543,7 +1623,13 @@ class DaemonApp:
             )
         turn_state = self._refresh_turn_worker_state(quest_id)
         has_live_turn = bool(turn_state.get("running"))
-        if runtime_status == "running" and has_live_turn:
+        stalled_details = self._stalled_running_turn_details(
+            quest_id,
+            snapshot=snapshot,
+            turn_state=turn_state,
+            turn_reason="user_message",
+        )
+        if runtime_status == "running" and has_live_turn and stalled_details is None:
             scheduled = {
                 "scheduled": True,
                 "started": False,
@@ -1712,18 +1798,30 @@ class DaemonApp:
         return snapshot
 
     def schedule_turn(self, quest_id: str, *, reason: str = "user_message") -> dict:
+        snapshot = self.quest_service.snapshot(quest_id)
+        snapshot = self._reconcile_stale_active_turn(quest_id, snapshot=snapshot)
+        recovery = self._recover_stalled_running_turn(quest_id, snapshot=snapshot, turn_reason=reason)
+        snapshot = dict(recovery.get("snapshot") or snapshot)
+        if recovery.get("blocked"):
+            return {
+                "scheduled": True,
+                "started": False,
+                "queued": True,
+                "reason": "stalled_turn_recovery_pending",
+            }
         self._refresh_turn_worker_state(quest_id)
         with self._turn_lock:
             state = self._turn_state.setdefault(quest_id, {"running": False, "pending": False})
             state["pending"] = True
             state["stop_requested"] = False
+            state.pop("recovery_pending", None)
             state["reason"] = reason
             if state.get("running"):
                 return {
                     "scheduled": True,
                     "started": False,
                     "queued": True,
-                    "reason": reason,
+                    "reason": "queued_for_artifact_interact" if reason == "user_message" else reason,
                 }
             state["running"] = True
             worker = threading.Thread(
@@ -1804,6 +1902,110 @@ class DaemonApp:
                 state.pop("worker", None)
             return dict(state)
 
+    def _wait_for_turn_worker_exit(self, quest_id: str, *, timeout_seconds: float) -> dict[str, object]:
+        deadline = time.monotonic() + max(0.0, float(timeout_seconds))
+        state = self._refresh_turn_worker_state(quest_id)
+        while state.get("running") and time.monotonic() < deadline:
+            time.sleep(0.05)
+            state = self._refresh_turn_worker_state(quest_id)
+        return state
+
+    def _ensure_recovery_resume_watch(self, quest_id: str, *, turn_reason: str) -> None:
+        with self._turn_lock:
+            state = self._turn_state.setdefault(quest_id, {"running": False, "pending": False})
+            if state.get("recovery_watch_active"):
+                return
+            state["recovery_watch_active"] = True
+        watcher = threading.Thread(
+            target=self._wait_and_resume_recovered_turn,
+            args=(quest_id, turn_reason),
+            daemon=True,
+            name=f"deepscientist-recovery-watch-{quest_id}",
+        )
+        watcher.start()
+
+    def _wait_and_resume_recovered_turn(self, quest_id: str, turn_reason: str) -> None:
+        try:
+            while True:
+                state = self._refresh_turn_worker_state(quest_id)
+                if not state.get("recovery_pending"):
+                    return
+                if not state.get("running"):
+                    break
+                time.sleep(0.1)
+
+            snapshot = self.quest_service.snapshot(quest_id)
+            runtime_status = str(snapshot.get("runtime_status") or snapshot.get("status") or "").strip().lower()
+            if runtime_status in {"paused", "stopped", "completed", "error"}:
+                with self._turn_lock:
+                    state = self._turn_state.setdefault(quest_id, {"running": False, "pending": False})
+                    state.pop("recovery_pending", None)
+                    state["stop_requested"] = runtime_status in {"paused", "stopped"}
+                return
+            pending_user_count = int(snapshot.get("pending_user_message_count") or 0)
+            if pending_user_count > 0:
+                with self._turn_lock:
+                    state = self._turn_state.setdefault(quest_id, {"running": False, "pending": False})
+                    state.pop("recovery_pending", None)
+                self.schedule_turn(quest_id, reason=turn_reason)
+                return
+
+            with self._turn_lock:
+                state = self._turn_state.setdefault(quest_id, {"running": False, "pending": False})
+                state.pop("recovery_pending", None)
+                state["stop_requested"] = False
+        except Exception as exc:
+            self.logger.log(
+                "warning",
+                "quest.turn_state_recovery_watch_failed",
+                quest_id=quest_id,
+                reason=turn_reason,
+                error=str(exc),
+            )
+        finally:
+            with self._turn_lock:
+                state = self._turn_state.setdefault(quest_id, {"running": False, "pending": False})
+                state.pop("recovery_watch_active", None)
+
+    def _stalled_running_turn_details(
+        self,
+        quest_id: str,
+        *,
+        snapshot: dict | None = None,
+        turn_state: dict[str, object] | None = None,
+        turn_reason: str,
+    ) -> dict[str, int] | None:
+        if str(turn_reason or "").strip() not in {"user_message", "queued_user_messages"}:
+            return None
+        snapshot = dict(snapshot or self.quest_service.snapshot(quest_id))
+        runtime_status = str(snapshot.get("runtime_status") or snapshot.get("status") or "").strip().lower()
+        active_run_id = str(snapshot.get("active_run_id") or "").strip()
+        if runtime_status != "running" or not active_run_id:
+            return None
+        state = dict(turn_state or self._refresh_turn_worker_state(quest_id))
+        if not state.get("running"):
+            return None
+        pending_user_count = int(snapshot.get("pending_user_message_count") or 0)
+        if pending_user_count <= 0:
+            return None
+        counts = snapshot.get("counts") if isinstance(snapshot.get("counts"), dict) else {}
+        if int(counts.get("bash_running_count") or 0) > 0:
+            return None
+        silent_seconds = snapshot.get("seconds_since_last_tool_activity")
+        if silent_seconds is None:
+            watchdog = snapshot.get("interaction_watchdog") if isinstance(snapshot.get("interaction_watchdog"), dict) else {}
+            silent_seconds = watchdog.get("seconds_since_last_tool_activity")
+        try:
+            silent_seconds_int = int(silent_seconds or 0)
+        except (TypeError, ValueError):
+            return None
+        if silent_seconds_int < _STALLED_RUNNING_TURN_INACTIVITY_SECONDS:
+            return None
+        return {
+            "pending_user_count": pending_user_count,
+            "silent_seconds": silent_seconds_int,
+        }
+
     def _reconcile_stale_active_turn(self, quest_id: str, *, snapshot: dict | None = None) -> dict:
         snapshot = dict(snapshot or self.quest_service.snapshot(quest_id))
         active_run_id = str(snapshot.get("active_run_id") or "").strip()
@@ -1855,6 +2057,139 @@ class DaemonApp:
         )
         return self.quest_service.mark_turn_finished(quest_id, status=normalized_status)
 
+    def _recover_stalled_running_turn(
+        self,
+        quest_id: str,
+        *,
+        snapshot: dict | None = None,
+        turn_reason: str,
+    ) -> dict[str, object]:
+        snapshot = dict(snapshot or self.quest_service.snapshot(quest_id))
+        turn_state = self._refresh_turn_worker_state(quest_id)
+        with self._turn_lock:
+            state = self._turn_state.setdefault(quest_id, {"running": False, "pending": False})
+            if state.get("recovery_pending"):
+                return {
+                    "snapshot": snapshot,
+                    "blocked": True,
+                }
+        details = self._stalled_running_turn_details(
+            quest_id,
+            snapshot=snapshot,
+            turn_state=turn_state,
+            turn_reason=turn_reason,
+        )
+        if details is None:
+            return {
+                "snapshot": snapshot,
+                "blocked": False,
+            }
+
+        active_run_id = str(snapshot.get("active_run_id") or "").strip()
+        runner_name = self._runner_name_for(snapshot)
+        with self._turn_lock:
+            state = self._turn_state.setdefault(quest_id, {"running": False, "pending": False})
+            if state.get("recovery_pending"):
+                return {
+                    "snapshot": snapshot,
+                    "blocked": True,
+                }
+            state["pending"] = False
+            state["stop_requested"] = True
+            state["recovery_pending"] = True
+        interrupted = False
+        try:
+            try:
+                runner = self.get_runner(runner_name)
+            except KeyError:
+                runner = None
+            if runner is not None and hasattr(runner, "interrupt"):
+                interrupted = bool(getattr(runner, "interrupt")(quest_id))
+            stopped_bash_session_ids = self._stop_active_bash_exec_sessions(
+                quest_id,
+                run_id=active_run_id or None,
+                reason="stalled_turn_recovery",
+                user_id="auto:stalled-turn-recovery",
+            )
+            turn_state = self._wait_for_turn_worker_exit(
+                quest_id,
+                timeout_seconds=_STALLED_RUNNING_TURN_INTERRUPT_TIMEOUT_SECONDS,
+            )
+            if turn_state.get("running"):
+                self._ensure_recovery_resume_watch(quest_id, turn_reason="queued_user_messages")
+                self.logger.log(
+                    "warning",
+                    "quest.turn_state_recovery_pending",
+                    quest_id=quest_id,
+                    abandoned_run_id=active_run_id or None,
+                    reason=turn_reason,
+                    silent_seconds=int(details.get("silent_seconds") or 0),
+                    pending_user_message_count=int(details.get("pending_user_count") or 0),
+                    interrupted=interrupted,
+                )
+                return {
+                    "snapshot": snapshot,
+                    "blocked": True,
+                }
+
+            previous_status = (
+                str(snapshot.get("runtime_status") or snapshot.get("status") or snapshot.get("display_status") or "running").strip()
+                or "running"
+            )
+            normalized_status = "active" if previous_status == "running" else previous_status
+            summary = (
+                f"Recovered stalled running turn `{active_run_id}` after "
+                f"{int(details.get('silent_seconds') or 0)} seconds without tool activity while "
+                f"{int(details.get('pending_user_count') or 0)} queued user message(s) were waiting."
+            )
+            if interrupted:
+                summary = f"{summary} The active runner process was interrupted."
+            if stopped_bash_session_ids:
+                summary = f"{summary} Stopped {len(stopped_bash_session_ids)} bash_exec session(s)."
+            quest_root = self.quest_service._quest_root(quest_id)
+            append_jsonl(
+                quest_root / ".ds" / "events.jsonl",
+                {
+                    "event_id": generate_id("evt"),
+                    "type": "quest.turn_state_reconciled",
+                    "quest_id": quest_id,
+                    "abandoned_run_id": active_run_id or None,
+                    "previous_status": previous_status,
+                    "status": normalized_status,
+                    "completed_at": None,
+                    "exit_code": None,
+                    "summary": summary,
+                    "recovery_kind": "stalled_live_turn",
+                    "interrupted": interrupted,
+                    "stopped_bash_session_ids": stopped_bash_session_ids,
+                    "created_at": utc_now(),
+                },
+            )
+            self.logger.log(
+                "warning",
+                "quest.turn_state_reconciled",
+                quest_id=quest_id,
+                abandoned_run_id=active_run_id or None,
+                previous_status=previous_status,
+                status=normalized_status,
+                recovery_kind="stalled_live_turn",
+                interrupted=interrupted,
+                stopped_bash_session_count=len(stopped_bash_session_ids),
+            )
+            snapshot = self.quest_service.mark_turn_finished(quest_id, status=normalized_status)
+            with self._turn_lock:
+                state = self._turn_state.setdefault(quest_id, {"running": False, "pending": False})
+                state.pop("recovery_pending", None)
+            return {
+                "snapshot": snapshot,
+                "blocked": False,
+            }
+        except Exception:
+            with self._turn_lock:
+                state = self._turn_state.setdefault(quest_id, {"running": False, "pending": False})
+                state.pop("recovery_pending", None)
+            raise
+
     def control_quest(self, quest_id: str, *, action: str, source: str = "local") -> dict:
         normalized_action = str(action or "").strip().lower()
         if normalized_action == "pause":
@@ -2321,12 +2656,15 @@ class DaemonApp:
             )
         retry_policy = self._runner_retry_policy(runner_name, runner_cfg if isinstance(runner_cfg, dict) else {})
         max_attempts = int(retry_policy.get("max_attempts") or 1)
-        turn_id = generate_id("turn")
-        retry_context: dict[str, Any] | None = None
+        resumed_start_attempt, resumed_turn_id, retry_context = self._resume_retry_state(
+            snapshot,
+            max_attempts=max_attempts,
+        )
+        turn_id = resumed_turn_id or generate_id("turn")
         quest_root = Path(snapshot["quest_root"])
         worktree_root = Path(str(snapshot["current_workspace_root"])) if snapshot.get("current_workspace_root") else None
 
-        for attempt_index in range(1, max_attempts + 1):
+        for attempt_index in range(resumed_start_attempt, max_attempts + 1):
             current_run_id = run_id if attempt_index == 1 else generate_id("run")
             if attempt_index > 1:
                 self._append_retry_event(
@@ -2395,6 +2733,31 @@ class DaemonApp:
                         previous_output_text="",
                         stderr_text=str(exc),
                     )
+                    diagnosis = self._non_retryable_failure_diagnosis(
+                        runner_name=runner_name,
+                        summary=failure_summary,
+                        stderr_text=str(exc),
+                        output_text="",
+                    )
+                    if diagnosis is not None:
+                        self.quest_service.update_runtime_state(
+                            quest_root=quest_root,
+                            continuation_policy="wait_for_user_or_resume",
+                            continuation_reason="non_retryable_runner_error",
+                            continuation_updated_at=utc_now(),
+                        )
+                        self._record_turn_error(
+                            quest_id=quest_id,
+                            runner_name=runner_name,
+                            run_id=current_run_id,
+                            skill_id=skill_id,
+                            model=model,
+                            summary=f"{diagnosis.problem} {failure_summary}".strip(),
+                            retry_state=None,
+                            diagnosis_code=diagnosis.code,
+                            guidance=list(diagnosis.guidance),
+                        )
+                        return
                     if bool(retry_policy.get("enabled")) and attempt_index < max_attempts:
                         delay_seconds = self._retry_delay_seconds(retry_policy, attempt_index=attempt_index + 1)
                         next_retry_at = self._retry_next_timestamp(delay_seconds)
@@ -2543,6 +2906,31 @@ class DaemonApp:
                     previous_output_text=result.output_text,
                     stderr_text=result.stderr_text,
                 )
+                diagnosis = self._non_retryable_failure_diagnosis(
+                    runner_name=runner_name,
+                    summary=failure_summary,
+                    stderr_text=result.stderr_text,
+                    output_text=result.output_text,
+                )
+                if diagnosis is not None:
+                    self.quest_service.update_runtime_state(
+                        quest_root=quest_root,
+                        continuation_policy="wait_for_user_or_resume",
+                        continuation_reason="non_retryable_runner_error",
+                        continuation_updated_at=utc_now(),
+                    )
+                    self._record_turn_error(
+                        quest_id=quest_id,
+                        runner_name=runner_name,
+                        run_id=result.run_id,
+                        skill_id=skill_id,
+                        model=model,
+                        summary=f"{diagnosis.problem} {failure_summary}".strip(),
+                        retry_state=None,
+                        diagnosis_code=diagnosis.code,
+                        guidance=list(diagnosis.guidance),
+                    )
+                    return
                 if bool(retry_policy.get("enabled")) and attempt_index < max_attempts:
                     delay_seconds = self._retry_delay_seconds(retry_policy, attempt_index=attempt_index + 1)
                     next_retry_at = self._retry_next_timestamp(delay_seconds)
@@ -2736,6 +3124,18 @@ class DaemonApp:
 
         return skill
 
+    @staticmethod
+    def _direct_user_turn_skill(snapshot: dict) -> str:
+        available_stage_skills = current_standard_skills(repo_root())
+        for candidate in (
+            str(snapshot.get("active_anchor") or "").strip(),
+            str(snapshot.get("continuation_anchor") or "").strip(),
+        ):
+            if candidate in available_stage_skills and candidate != "decision":
+                return DaemonApp._turn_skill_stage_gate(snapshot, candidate)
+        fallback = "baseline" if "baseline" in available_stage_skills else "scout"
+        return DaemonApp._turn_skill_stage_gate(snapshot, fallback)
+
     @staticmethod
     def _turn_skill_for(
         snapshot: dict,
@@ -2747,16 +3147,6 @@ class DaemonApp:
         available_stage_skills = current_standard_skills(repo_root())
         workspace_mode = DaemonApp._workspace_mode_for(snapshot)
 
-        def copilot_default_skill() -> str:
-            active_anchor = str(snapshot.get("active_anchor") or "").strip()
-            if active_anchor in available_stage_skills and active_anchor != "decision":
-                return DaemonApp._turn_skill_stage_gate(snapshot, active_anchor)
-            continuation_anchor = str(snapshot.get("continuation_anchor") or "").strip()
-            if continuation_anchor in available_stage_skills and continuation_anchor != "decision":
-                return DaemonApp._turn_skill_stage_gate(snapshot, continuation_anchor)
-            fallback = "baseline" if "baseline" in available_stage_skills else "scout"
-            return DaemonApp._turn_skill_stage_gate(snapshot, fallback)
-
         reply_target = str((latest_user_message or {}).get("reply_to_interaction_id") or "").strip()
         if reply_target:
             for item in (snapshot.get("active_interactions") or []):
@@ -2783,8 +3173,8 @@ class DaemonApp:
                 ):
                     return "decision"
                 if str(item.get("reply_mode") or "") == "threaded":
-                    if workspace_mode == "copilot":
-                        return copilot_default_skill()
+                    if workspace_mode == "copilot" or turn_mode in {"answering", "command_execution"}:
+                        return DaemonApp._direct_user_turn_skill(snapshot)
                     return DaemonApp._turn_skill_stage_gate(
                         snapshot,
                         DaemonApp._continuation_anchor_for(snapshot),
@@ -2792,9 +3182,9 @@ class DaemonApp:
         if turn_mode == "recovering":
             return "decision"
         if workspace_mode == "copilot" and latest_user_message is not None:
-            return copilot_default_skill()
+            return DaemonApp._direct_user_turn_skill(snapshot)
         if turn_mode in {"answering", "command_execution"}:
-            return "decision"
+            return DaemonApp._direct_user_turn_skill(snapshot)
         if str(turn_reason or "").strip() == "auto_continue" or latest_user_message is None:
             return DaemonApp._turn_skill_stage_gate(
                 snapshot,
@@ -3108,8 +3498,11 @@ class DaemonApp:
         summary: str,
         display_status: str = "error",
         retry_state: dict[str, Any] | None = None,
+        diagnosis_code: str | None = None,
+        guidance: list[str] | None = None,
     ) -> None:
         quest_root = self.home / "quests" / quest_id
+        normalized_guidance = [str(line) for line in (guidance or []) if str(line).strip()]
         append_jsonl(
             quest_root / ".ds" / "events.jsonl",
             {
@@ -3121,6 +3514,8 @@ class DaemonApp:
                 "skill_id": skill_id,
                 "model": model,
                 "summary": summary,
+                "diagnosis_code": str(diagnosis_code or "").strip() or None,
+                "guidance": normalized_guidance,
                 "created_at": utc_now(),
             },
         )
@@ -3131,6 +3526,16 @@ class DaemonApp:
             active_run_id=None,
             retry_state=retry_state,
         )
+        notice_message = summary
+        if normalized_guidance:
+            notice_message = "\n".join(
+                [
+                    summary,
+                    "",
+                    "Suggested fix:",
+                    *[f"- {line}" for line in normalized_guidance[:3]],
+                ]
+            ).strip()
         self.logger.log(
             "error",
             "runner.turn_error",
@@ -3143,7 +3548,7 @@ class DaemonApp:
         )
         self._relay_quest_message_to_bound_connectors(
             quest_id,
-            message=summary,
+            message=notice_message,
             kind="error",
             response_phase="final",
             importance="warning",
@@ -3154,10 +3559,29 @@ class DaemonApp:
                     "skill_id": skill_id,
                     "runner": runner_name,
                     "model": model,
+                    "diagnosis_code": str(diagnosis_code or "").strip() or None,
                 }
             ],
         )
 
+    @staticmethod
+    def _non_retryable_failure_diagnosis(
+        *,
+        runner_name: str,
+        summary: str,
+        stderr_text: str,
+        output_text: str,
+    ) -> FailureDiagnosis | None:
+        diagnosis = diagnose_runner_failure(
+            runner_name=runner_name,
+            summary=summary,
+            stderr_text=stderr_text,
+            output_text=output_text,
+        )
+        if diagnosis is None or diagnosis.retriable:
+            return None
+        return diagnosis
+
     def _record_turn_postprocess_warning(
         self,
         *,
@@ -7375,7 +7799,7 @@ class DaemonApp:
                         payload = result(**params, path=self.path)
                     elif method == "GET":
                         payload = result(**params) if params else result()
-                    elif route_name in {"document_open", "document_asset_upload", "chat", "command", "quest_control", "config_save", "quest_create", "quest_baseline_binding", "run_create", "qq_inbound", "connector_inbound", "docs_open", "admin_shutdown", "bash_stop", "quest_settings", "quest_bindings", "quest_delete", "quest_layout_update", "terminal_session_ensure", "terminal_attach", "terminal_input", "stage_view", "latex_init", "latex_compile", "system_update_action", "weixin_login_qr_start", "weixin_login_qr_wait", "arxiv_import", "annotation_create", "auth_login", "auth_rotate"}:
+                    elif route_name in {"document_open", "document_asset_upload", "quest_file_create_folder", "quest_file_upload", "quest_file_rename", "quest_file_move", "quest_file_delete", "chat", "command", "quest_control", "config_save", "quest_create", "quest_baseline_binding", "run_create", "qq_inbound", "connector_inbound", "docs_open", "admin_shutdown", "bash_stop", "quest_settings", "quest_bindings", "quest_delete", "quest_layout_update", "terminal_session_ensure", "terminal_attach", "terminal_input", "stage_view", "latex_init", "latex_compile", "system_update_action", "weixin_login_qr_start", "weixin_login_qr_wait", "arxiv_import", "annotation_create", "auth_login", "auth_rotate"}:
                         payload = result(**params, body=body)
                     elif route_name == "config_validate":
                         payload = result(body)

package/src/deepscientist/diagnostics/__init__.py

@@ -0,0 +1,6 @@
+from .runner_failures import FailureDiagnosis, diagnose_runner_failure
+
+__all__ = [
+    "FailureDiagnosis",
+    "diagnose_runner_failure",
+]