@researai/deepscientist 1.5.15 → 1.5.17

package/src/deepscientist/quest/service.py

@@ -5,10 +5,10 @@ from collections import deque
 from contextlib import contextmanager
 from datetime import UTC, datetime, timedelta
 import hashlib
-import subprocess
 import json
 import mimetypes
 import re
+import shutil
 import threading
 import time
 from pathlib import Path, PurePosixPath
@@ -24,10 +24,10 @@ from ..artifact.metrics import build_baseline_compare_payload, build_metrics_tim
 from ..config import ConfigManager
 from ..connector_runtime import conversation_identity_key, normalize_conversation_id, parse_conversation_id
 from ..file_lock import advisory_file_lock
-from ..gitops import current_branch, export_git_graph, head_commit, init_repo, list_branch_canvas
+from ..gitops import current_branch, export_git_graph, head_commit, init_repo, list_branch_canvas, list_commit_canvas
 from ..home import repo_root
 from ..registries import BaselineRegistry
-from ..shared import append_jsonl, ensure_dir, generate_id, iter_jsonl, read_json, read_jsonl, read_jsonl_tail, read_text, read_yaml, resolve_within, run_command, sha256_text, slugify, utc_now, write_json, write_text, write_yaml
+from ..shared import append_jsonl, ensure_dir, generate_id, iter_jsonl, read_json, read_jsonl, read_jsonl_tail, read_text, read_yaml, resolve_within, run_command, run_command_bytes, sha256_text, slugify, utc_now, write_json, write_text, write_yaml
 from ..skills import SkillInstaller
 from ..web_search import extract_web_search_payload
 from .layout import (
@@ -172,6 +172,127 @@ def _iter_jsonl_records_safely(
                     yield payload
 
 
+def _parse_jsonl_record_line_safely(
+    raw_line: bytes,
+    *,
+    oversized_line_bytes: int = _EVENTS_OVERSIZED_LINE_BYTES,
+) -> dict[str, Any] | None:
+    raw = bytes(raw_line).strip()
+    if not raw:
+        return None
+    line_bytes = len(raw)
+    if line_bytes > oversized_line_bytes:
+        return _oversized_event_placeholder(
+            prefix=raw[:_OVERSIZED_EVENT_PREFIX_BYTES],
+            line_bytes=line_bytes,
+        )
+    try:
+        payload = json.loads(raw)
+    except json.JSONDecodeError:
+        return None
+    return payload if isinstance(payload, dict) else None
+
+
+def _tail_jsonl_records_safely(
+    path: Path,
+    *,
+    limit: int,
+    oversized_line_bytes: int = _EVENTS_OVERSIZED_LINE_BYTES,
+) -> tuple[list[tuple[int, dict[str, Any]]], int]:
+    normalized_limit = max(int(limit or 0), 0)
+    if normalized_limit <= 0 or not path.exists():
+        return [], 0
+    total = _count_jsonl_lines_fast(path)
+    if total <= 0:
+        return [], 0
+
+    raw_tail = _read_jsonl_tail_lines_fast(path, normalized_limit)
+    if not raw_tail:
+        return [], total
+
+    cursor_start = max(total - len(raw_tail) + 1, 1)
+    parsed: list[tuple[int, dict[str, Any]]] = []
+    for cursor, raw_line in enumerate(raw_tail, start=cursor_start):
+        payload = _parse_jsonl_record_line_safely(
+            raw_line,
+            oversized_line_bytes=oversized_line_bytes,
+        )
+        if isinstance(payload, dict):
+            parsed.append((cursor, payload))
+    return parsed, total
+
+
+def _count_jsonl_lines_fast(path: Path, *, chunk_size: int = 1024 * 1024) -> int:
+    if not path.exists():
+        return 0
+    total = 0
+    last_byte = b""
+    with path.open("rb") as handle:
+        while True:
+            chunk = handle.read(chunk_size)
+            if not chunk:
+                break
+            total += chunk.count(b"\n")
+            last_byte = chunk[-1:]
+    if total == 0 and last_byte:
+        return 1
+    if last_byte not in {b"", b"\n"}:
+        total += 1
+    return total
+
+
+def _read_jsonl_tail_lines_fast(path: Path, limit: int, *, chunk_size: int = 1024 * 1024) -> list[bytes]:
+    normalized_limit = max(int(limit or 0), 0)
+    if normalized_limit <= 0 or not path.exists():
+        return []
+
+    size = path.stat().st_size
+    if size <= 0:
+        return []
+
+    lines: deque[bytes] = deque()
+    remainder = b""
+    with path.open("rb") as handle:
+        position = size
+        while position > 0 and len(lines) < normalized_limit:
+            read_size = min(chunk_size, position)
+            position -= read_size
+            handle.seek(position)
+            chunk = handle.read(read_size)
+            payload = chunk + remainder
+            parts = payload.split(b"\n")
+            remainder = parts[0]
+            for raw_line in reversed(parts[1:]):
+                stripped = raw_line.rstrip(b"\r")
+                if not stripped.strip():
+                    continue
+                lines.appendleft(stripped)
+                if len(lines) >= normalized_limit:
+                    break
+        if len(lines) < normalized_limit and remainder.strip():
+            lines.appendleft(remainder.rstrip(b"\r"))
+    return list(lines)[-normalized_limit:]
+
+
+def _iter_jsonl_records_from_offset_safely(
+    path: Path,
+    *,
+    start_offset: int,
+    oversized_line_bytes: int = _EVENTS_OVERSIZED_LINE_BYTES,
+):
+    if not path.exists():
+        return
+    with path.open("rb") as handle:
+        handle.seek(max(int(start_offset or 0), 0))
+        for raw_line in handle:
+            payload = _parse_jsonl_record_line_safely(
+                raw_line,
+                oversized_line_bytes=oversized_line_bytes,
+            )
+            if isinstance(payload, dict):
+                yield payload
+
+
 class QuestService:
     def __init__(self, home: Path, skill_installer: SkillInstaller | None = None) -> None:
         self.home = home
@@ -182,6 +303,7 @@ class QuestService:
         self._file_cache: dict[str, dict[str, Any]] = {}
         self._jsonl_cache_lock = threading.Lock()
         self._jsonl_cache: dict[str, dict[str, Any]] = {}
+        self._jsonl_tail_cache: dict[str, dict[str, Any]] = {}
         self._snapshot_cache_lock = threading.Lock()
         self._snapshot_cache: dict[str, dict[str, Any]] = {}
         self._codex_history_cache_lock = threading.Lock()
@@ -200,6 +322,12 @@ class QuestService:
     def _quest_root(self, quest_id: str) -> Path:
         return self.quests_root / quest_id
 
+    def _require_initialized_quest_root(self, quest_id: str) -> Path:
+        quest_root = self._quest_root(quest_id)
+        if not quest_root.exists() or not self._quest_yaml_path(quest_root).exists():
+            raise FileNotFoundError(f"Unknown quest `{quest_id}`.")
+        return quest_root
+
     def _normalized_binding_sources(self, sources: list[Any] | None) -> list[str]:
         local_present = False
         external_source: str | None = None
@@ -288,6 +416,13 @@ class QuestService:
         return quest_root / ".ds" / "lab_canvas_state.json"
 
     def _default_research_state(self, quest_root: Path) -> dict[str, Any]:
+        quest_yaml = self.read_quest_yaml(quest_root)
+        startup_contract = (
+            dict(quest_yaml.get("startup_contract") or {})
+            if isinstance(quest_yaml.get("startup_contract"), dict)
+            else {}
+        )
+        workspace_mode = str(startup_contract.get("workspace_mode") or "").strip().lower() or "quest"
         return {
             "version": 1,
             "active_idea_id": None,
@@ -304,7 +439,7 @@ class QuestService:
             "paper_parent_worktree_root": None,
             "paper_parent_run_id": None,
             "next_pending_slice_id": None,
-            "workspace_mode": "quest",
+            "workspace_mode": workspace_mode,
             "last_flow_type": None,
             "updated_at": utc_now(),
         }
@@ -354,7 +489,7 @@ class QuestService:
                 continue
             current[key] = str(value) if isinstance(value, Path) else value
         payload = self.write_research_state(quest_root, current)
-        self.schedule_projection_refresh(quest_root, kinds=("details", "canvas"))
+        self.schedule_projection_refresh(quest_root, kinds=("details", "canvas", "git_canvas"))
         return payload
 
     def read_lab_canvas_state(self, quest_root: Path) -> dict[str, Any]:
@@ -971,6 +1106,8 @@ class QuestService:
             return self._details_projection_state(quest_root)
         if kind == "canvas":
             return self._canvas_projection_state(quest_root)
+        if kind == "git_canvas":
+            return self._canvas_projection_state(quest_root)
         raise ValueError(f"Unsupported projection kind `{kind}`.")
 
     def _projection_source_signature(self, quest_root: Path, kind: str) -> str:
@@ -1434,6 +1571,17 @@ class QuestService:
         update_progress(2, "Computing branch canvas")
         return list_branch_canvas(quest_root, quest_id=quest_root.name)
 
+    def _build_git_canvas_projection_payload(
+        self,
+        quest_root: Path,
+        *,
+        source_signature: str,
+        update_progress: Any,
+    ) -> dict[str, Any]:
+        update_progress(1, "Scanning commit history")
+        update_progress(2, "Computing commit canvas")
+        return list_commit_canvas(quest_root, quest_id=quest_root.name)
+
     def _build_projection_payload(
         self,
         quest_root: Path,
@@ -1454,6 +1602,12 @@ class QuestService:
                 source_signature=source_signature,
                 update_progress=update_progress,
             )
+        if kind == "git_canvas":
+            return self._build_git_canvas_projection_payload(
+                quest_root,
+                source_signature=source_signature,
+                update_progress=update_progress,
+            )
         raise ValueError(f"Unsupported projection kind `{kind}`.")
 
     def _placeholder_workflow_payload(self, quest_id: str, quest_root: Path) -> dict[str, Any]:
@@ -1486,6 +1640,17 @@ class QuestService:
             },
         }
 
+    def _placeholder_git_canvas_payload(self, quest_id: str, quest_root: Path) -> dict[str, Any]:
+        research_state = self.read_research_state(quest_root)
+        return {
+            "quest_id": quest_id,
+            "workspace_mode": str(research_state.get("workspace_mode") or "copilot").strip() or "copilot",
+            "head": head_commit(quest_root),
+            "current_ref": current_branch(quest_root),
+            "nodes": [],
+            "edges": [],
+        }
+
     def _projected_payload(self, quest_id: str, kind: str) -> dict[str, Any]:
         quest_root = self._quest_root(quest_id)
         source_signature = self._projection_source_signature(quest_root, kind)
@@ -1510,11 +1675,12 @@ class QuestService:
             else None
         )
         if payload is None:
-            payload = (
-                self._placeholder_workflow_payload(quest_id, quest_root)
-                if kind == "details"
-                else self._placeholder_canvas_payload(quest_id, quest_root)
-            )
+            if kind == "details":
+                payload = self._placeholder_workflow_payload(quest_id, quest_root)
+            elif kind == "git_canvas":
+                payload = self._placeholder_git_canvas_payload(quest_id, quest_root)
+            else:
+                payload = self._placeholder_canvas_payload(quest_id, quest_root)
         payload["projection_status"] = status
         return payload
 
@@ -1535,8 +1701,8 @@ class QuestService:
     ) -> None:
         resolved_kinds = [
             str(kind).strip()
-            for kind in (kinds or ("details", "canvas"))
-            if str(kind).strip() in {"details", "canvas"}
+            for kind in (kinds or ("details", "canvas", "git_canvas"))
+            if str(kind).strip() in {"details", "canvas", "git_canvas"}
         ]
         if not resolved_kinds:
             return
@@ -1563,6 +1729,9 @@ class QuestService:
     def git_branch_canvas(self, quest_id: str) -> dict[str, Any]:
         return self._projected_payload(quest_id, "canvas")
 
+    def git_commit_canvas(self, quest_id: str) -> dict[str, Any]:
+        return self._projected_payload(quest_id, "git_canvas")
+
     def _active_baseline_attachment(self, quest_root: Path, workspace_root: Path) -> dict[str, Any] | None:
         attachments: list[dict[str, Any]] = []
         seen_paths: set[str] = set()
@@ -2602,7 +2771,7 @@ class QuestService:
         )
         write_text(quest_root / "brief.md", initial_brief(goal))
         write_text(quest_root / "plan.md", initial_plan())
-        write_text(quest_root / "status.md", initial_status())
+        write_text(quest_root / "status.md", initial_status(startup_contract))
         write_text(quest_root / "SUMMARY.md", initial_summary())
         write_text(quest_root / ".gitignore", gitignore())
         self._write_active_user_requirements(
@@ -2619,6 +2788,86 @@ class QuestService:
         self._initialize_runtime_files(quest_root)
         return self.snapshot(quest_id)
 
+    def repair_orphaned_quest_scaffold(
+        self,
+        quest_id: str,
+        *,
+        title: str | None = None,
+        goal: str | None = None,
+        runner: str = "codex",
+    ) -> dict[str, Any]:
+        quest_root = self._quest_root(quest_id)
+        if not quest_root.exists():
+            raise FileNotFoundError(f"Unknown quest `{quest_id}`.")
+        quest_yaml_path = self._quest_yaml_path(quest_root)
+        if quest_yaml_path.exists():
+            raise FileExistsError(f"Quest `{quest_id}` already has a scaffold.")
+
+        restored_goal = str(goal or f"Recovered quest {quest_id}").strip() or f"Recovered quest {quest_id}"
+        restored_title = str(title or quest_id).strip() or quest_id
+
+        for relative in QUEST_DIRECTORIES:
+            ensure_dir(quest_root / relative)
+
+        write_yaml(
+            quest_yaml_path,
+            initial_quest_yaml(
+                quest_id,
+                restored_goal,
+                quest_root,
+                runner,
+                title=restored_title,
+            ),
+        )
+        write_text(
+            quest_root / "brief.md",
+            "\n".join(
+                [
+                    "# Quest Brief",
+                    "",
+                    "## Recovery Note",
+                    "",
+                    "This quest scaffold was recreated because the core quest files were missing.",
+                    "Existing runtime traces under `.ds/` were preserved.",
+                    "",
+                    "## Goal",
+                    "",
+                    restored_goal,
+                    "",
+                ]
+            ),
+        )
+        write_text(
+            quest_root / "plan.md",
+            "\n".join(
+                [
+                    "# Plan",
+                    "",
+                    "- [ ] Inspect preserved runtime traces under `.ds/`",
+                    "- [ ] Re-establish the baseline context",
+                    "- [ ] Recreate any missing durable files or artifacts",
+                    "",
+                ]
+            ),
+        )
+        write_text(
+            quest_root / "status.md",
+            "# Status\n\nRecovered scaffold. Review preserved runtime state before continuing.\n",
+        )
+        write_text(
+            quest_root / "SUMMARY.md",
+            "# Summary\n\nRecovered quest scaffold. Original top-level quest files were missing.\n",
+        )
+        write_text(quest_root / ".gitignore", gitignore())
+        self._write_active_user_requirements(
+            quest_root,
+            latest_requirement=None,
+        )
+        if not (quest_root / ".git").exists():
+            init_repo(quest_root)
+        self._initialize_runtime_files(quest_root)
+        return self.snapshot(quest_id)
+
     def list_quests(self) -> list[dict]:
         items: list[dict] = []
         if not self.quests_root.exists():
@@ -2716,7 +2965,7 @@ class QuestService:
         )
 
     def summary_compact(self, quest_id: str) -> dict[str, Any]:
-        quest_root = self._quest_root(quest_id)
+        quest_root = self._require_initialized_quest_root(quest_id)
         cache_key = f"compact:{self._cache_key_for_path(quest_root)}"
         state = self._compact_summary_state(quest_root)
         with self._snapshot_cache_lock:
@@ -2790,6 +3039,7 @@ class QuestService:
             "research_head_worktree_root": research_state.get("research_head_worktree_root"),
             "current_workspace_branch": research_state.get("current_workspace_branch"),
             "current_workspace_root": research_state.get("current_workspace_root"),
+            "workspace_mode": research_state.get("workspace_mode") or "quest",
             "active_idea_id": research_state.get("active_idea_id"),
             "active_baseline_id": active_baseline_id,
             "active_baseline_variant_id": active_baseline_variant_id,
@@ -2876,8 +3126,8 @@ class QuestService:
             }
         return items
 
-    @staticmethod
     def _read_jsonl_cursor_slice(
+        self,
         path: Path,
         *,
         after: int = 0,
@@ -2886,7 +3136,10 @@ class QuestService:
         tail: bool = False,
     ) -> tuple[list[tuple[int, dict[str, Any]]], int, bool]:
         normalized_limit = max(int(limit or 0), 0)
+        cache_key = self._cache_key_for_path(path)
         if not path.exists():
+            with self._jsonl_cache_lock:
+                self._jsonl_tail_cache.pop(cache_key, None)
             return [], 0, False
         if normalized_limit <= 0:
             total = sum(1 for _ in _iter_jsonl_records_safely(path))
@@ -2905,11 +3158,71 @@ class QuestService:
             return list(window), total, has_more
 
         if tail:
-            window = deque(maxlen=normalized_limit)
-            total = 0
-            for payload in _iter_jsonl_records_safely(path):
-                total += 1
-                window.append((total, payload))
+            state = self._path_state(path)
+            cached_tail: dict[str, Any] | None = None
+            with self._jsonl_cache_lock:
+                candidate = self._jsonl_tail_cache.get(cache_key)
+                if isinstance(candidate, dict):
+                    cached_tail = dict(candidate)
+
+            if cached_tail and cached_tail.get("state") == state:
+                cached_limit = int(cached_tail.get("limit") or 0)
+                cached_records = list(cached_tail.get("records") or [])
+                cached_total = int(cached_tail.get("total") or 0)
+                if cached_limit >= normalized_limit and cached_records:
+                    window = cached_records[-normalized_limit:]
+                    has_more = cached_total > len(window)
+                    return window, cached_total, has_more
+
+            if (
+                cached_tail
+                and state is not None
+                and cached_tail.get("state")
+                and tuple(cached_tail.get("state"))[0] == state[0]
+                and state[2] >= tuple(cached_tail.get("state"))[2]
+            ):
+                cached_state = tuple(cached_tail.get("state"))
+                cached_limit = int(cached_tail.get("limit") or 0)
+                cached_total = int(cached_tail.get("total") or 0)
+                max_limit = max(normalized_limit, cached_limit)
+                window = deque(
+                    list(cached_tail.get("records") or []),
+                    maxlen=max_limit,
+                )
+                appended_records = list(
+                    _iter_jsonl_records_from_offset_safely(
+                        path,
+                        start_offset=int(cached_state[2]),
+                    )
+                )
+                if appended_records:
+                    next_cursor = cached_total + 1
+                    for payload in appended_records:
+                        window.append((next_cursor, payload))
+                        next_cursor += 1
+                    total = cached_total + len(appended_records)
+                else:
+                    total = cached_total
+                stored_records = list(window)
+                with self._jsonl_cache_lock:
+                    self._jsonl_tail_cache[cache_key] = {
+                        "state": state,
+                        "limit": max_limit,
+                        "total": total,
+                        "records": stored_records,
+                    }
+                selected = stored_records[-normalized_limit:]
+                has_more = total > len(selected)
+                return selected, total, has_more
+
+            window, total = _tail_jsonl_records_safely(path, limit=normalized_limit)
+            with self._jsonl_cache_lock:
+                self._jsonl_tail_cache[cache_key] = {
+                    "state": state,
+                    "limit": normalized_limit,
+                    "total": total,
+                    "records": list(window),
+                }
             has_more = total > len(window)
             return list(window), total, has_more
 
@@ -2945,6 +3258,14 @@ class QuestService:
         except FileNotFoundError:
             return str(path.absolute())
 
+    def jsonl_tail_cache_entry(self, path: Path) -> dict[str, Any] | None:
+        cache_key = self._cache_key_for_path(path)
+        with self._jsonl_cache_lock:
+            candidate = self._jsonl_tail_cache.get(cache_key)
+            if isinstance(candidate, dict):
+                return dict(candidate)
+        return None
+
     def _read_cached_path(
         self,
         path: Path,
@@ -3019,7 +3340,7 @@ class QuestService:
         return self._snapshot(quest_id)
 
     def _snapshot(self, quest_id: str) -> dict:
-        quest_root = self._quest_root(quest_id)
+        quest_root = self._require_initialized_quest_root(quest_id)
         cache_key = f"snapshot:{self._cache_key_for_path(quest_root)}"
         state = self._snapshot_state(quest_root)
         with self._snapshot_cache_lock:
@@ -3589,6 +3910,7 @@ class QuestService:
         title: str | None = None,
         active_anchor: str | None = None,
         default_runner: str | None = None,
+        workspace_mode: str | None = None,
     ) -> dict:
         quest_root = self._quest_root(quest_id)
         quest_yaml_path = self._quest_yaml_path(quest_root)
@@ -3597,6 +3919,8 @@ class QuestService:
 
         quest_data = self.read_quest_yaml(quest_root)
         changed = False
+        research_state_updates: dict[str, Any] = {}
+        runtime_state_updates: dict[str, Any] = {}
 
         if title is not None:
             normalized_title = str(title).strip()
@@ -3610,10 +3934,11 @@ class QuestService:
             normalized_anchor = str(active_anchor).strip()
             if not normalized_anchor:
                 raise ValueError("`active_anchor` cannot be empty.")
-            from ..prompts.builder import STANDARD_SKILLS
+            from ..prompts.builder import current_standard_skills
 
-            if normalized_anchor not in STANDARD_SKILLS:
-                allowed = ", ".join(STANDARD_SKILLS)
+            available_stage_skills = current_standard_skills(repo_root())
+            if normalized_anchor not in available_stage_skills:
+                allowed = ", ".join(available_stage_skills)
                 raise ValueError(f"Unsupported active anchor `{normalized_anchor}`. Allowed values: {allowed}.")
             if quest_data.get("active_anchor") != normalized_anchor:
                 quest_data["active_anchor"] = normalized_anchor
@@ -3633,9 +3958,35 @@ class QuestService:
                 quest_data["default_runner"] = normalized_runner
                 changed = True
 
+        if workspace_mode is not None:
+            normalized_workspace_mode = str(workspace_mode).strip().lower()
+            if normalized_workspace_mode not in {"copilot", "autonomous"}:
+                raise ValueError("Unsupported workspace mode. Allowed values: copilot, autonomous.")
+            startup_contract = (
+                dict(quest_data.get("startup_contract") or {})
+                if isinstance(quest_data.get("startup_contract"), dict)
+                else {}
+            )
+            if str(startup_contract.get("workspace_mode") or "").strip().lower() != normalized_workspace_mode:
+                startup_contract["workspace_mode"] = normalized_workspace_mode
+                quest_data["startup_contract"] = startup_contract
+                changed = True
+            if str(self.read_research_state(quest_root).get("workspace_mode") or "").strip().lower() != normalized_workspace_mode:
+                research_state_updates["workspace_mode"] = normalized_workspace_mode
+            runtime_state_updates["continuation_policy"] = (
+                "wait_for_user_or_resume" if normalized_workspace_mode == "copilot" else "auto"
+            )
+            runtime_state_updates["continuation_reason"] = (
+                "copilot_mode" if normalized_workspace_mode == "copilot" else "autonomous_mode"
+            )
+
         if changed:
             quest_data["updated_at"] = utc_now()
             write_yaml(quest_yaml_path, quest_data)
+        if research_state_updates:
+            self.update_research_state(quest_root, **research_state_updates)
+        if runtime_state_updates:
+            self.update_runtime_state(quest_root=quest_root, **runtime_state_updates)
 
         return self.snapshot(quest_id)
 
@@ -3670,10 +4021,11 @@ class QuestService:
             normalized_anchor = str(active_anchor or "").strip()
             if not normalized_anchor:
                 raise ValueError("`active_anchor` cannot be empty.")
-            from ..prompts.builder import STANDARD_SKILLS
+            from ..prompts.builder import current_standard_skills
 
-            if normalized_anchor not in STANDARD_SKILLS:
-                allowed = ", ".join(STANDARD_SKILLS)
+            available_stage_skills = current_standard_skills(repo_root())
+            if normalized_anchor not in available_stage_skills:
+                allowed = ", ".join(available_stage_skills)
                 raise ValueError(f"Unsupported active anchor `{normalized_anchor}`. Allowed values: {allowed}.")
             if quest_data.get("active_anchor") != normalized_anchor:
                 quest_data["active_anchor"] = normalized_anchor
@@ -4071,7 +4423,7 @@ class QuestService:
             return payload
 
     def list_documents(self, quest_id: str) -> list[dict]:
-        quest_root = self._quest_root(quest_id)
+        quest_root = self._require_initialized_quest_root(quest_id)
         workspace_root = self.active_workspace_root(quest_root)
         documents = []
         for relative in ("brief.md", "plan.md", "status.md", "SUMMARY.md"):
@@ -4125,7 +4477,7 @@ class QuestService:
         if revision:
             return self._revision_explorer(quest_id, revision=revision, mode=mode or "ref")
 
-        quest_root = self._quest_root(quest_id)
+        quest_root = self._require_initialized_quest_root(quest_id)
         workspace_root = self.active_workspace_root(quest_root)
         git_status = self._git_status_map(workspace_root)
 
@@ -4154,7 +4506,7 @@ class QuestService:
     def search_files(self, quest_id: str, term: str, limit: int = 50) -> dict[str, Any]:
         query = term.strip()
         normalized_query = query.casefold()
-        workspace_root = self.active_workspace_root(self._quest_root(quest_id))
+        workspace_root = self.active_workspace_root(self._require_initialized_quest_root(quest_id))
         resolved_limit = max(1, min(limit, 200))
         if not normalized_query:
             return {
@@ -4249,7 +4601,7 @@ class QuestService:
         }
 
     def open_document(self, quest_id: str, document_id: str) -> dict:
-        quest_root = self._quest_root(quest_id)
+        quest_root = self._require_initialized_quest_root(quest_id)
         workspace_root = self.active_workspace_root(quest_root)
         if document_id.startswith("git::"):
             revision, relative = self._parse_git_document_id(document_id)
@@ -4285,23 +4637,7 @@ class QuestService:
                 },
             }
 
-        resolution_root = (
-            quest_root
-            if document_id.startswith(("questpath::", "memory::"))
-            else workspace_root
-        )
-        try:
-            path, writable, scope, source_kind = self._resolve_document(resolution_root, document_id)
-        except FileNotFoundError:
-            legacy_relative = None
-            if document_id.startswith("path::"):
-                legacy_relative = document_id.split("::", 1)[1].lstrip("/")
-            if legacy_relative and legacy_relative.startswith("literature/arxiv/"):
-                path, writable, scope, source_kind = self._resolve_document(
-                    quest_root, f"questpath::{legacy_relative}"
-                )
-            else:
-                raise
+        path, writable, scope, source_kind = self.resolve_document(quest_id, document_id)
         renderer_hint, mime_type = self._renderer_hint_for(path)
         is_text = self._is_text_document(path, mime_type, renderer_hint)
         content = read_text(path) if is_text else ""
@@ -4329,6 +4665,24 @@ class QuestService:
             },
         }
 
+    def resolve_document(self, quest_id: str, document_id: str) -> tuple[Path, bool, str, str]:
+        quest_root = self._require_initialized_quest_root(quest_id)
+        workspace_root = self.active_workspace_root(quest_root)
+        resolution_root = self._document_resolution_root(
+            quest_root=quest_root,
+            workspace_root=workspace_root,
+            document_id=document_id,
+        )
+        try:
+            return self._resolve_document(resolution_root, document_id)
+        except FileNotFoundError:
+            legacy_relative = None
+            if document_id.startswith("path::"):
+                legacy_relative = document_id.split("::", 1)[1].lstrip("/")
+            if legacy_relative and legacy_relative.startswith("literature/arxiv/"):
+                return self._resolve_document(quest_root, f"questpath::{legacy_relative}")
+            raise
+
     def save_document(self, quest_id: str, document_id: str, content: str, previous_revision: str | None = None) -> dict:
         current = self.open_document(quest_id, document_id)
         if not current.get("writable", False):
@@ -4505,6 +4859,291 @@ class QuestService:
             "saved_at": utc_now(),
         }
 
+    @staticmethod
+    def _normalize_workspace_relative_path(
+        relative: str | None,
+        *,
+        field_name: str,
+        allow_root: bool = True,
+    ) -> str | None:
+        if relative is None:
+            if allow_root:
+                return None
+            raise ValueError(f"`{field_name}` is required.")
+        raw = str(relative).strip().replace("\\", "/")
+        if not raw:
+            if allow_root:
+                return None
+            raise ValueError(f"`{field_name}` is required.")
+        normalized = raw.lstrip("/").rstrip("/")
+        if normalized in {"", "."}:
+            if allow_root:
+                return None
+            raise ValueError(f"`{field_name}` must point to a workspace entry.")
+        return normalized
+
+    @staticmethod
+    def _normalize_workspace_entry_name(name: str | None, *, field_name: str) -> str:
+        raw = str(name or "").strip().replace("\\", "/")
+        if not raw:
+            raise ValueError(f"`{field_name}` is required.")
+        if "/" in raw:
+            raise ValueError(f"`{field_name}` must be a single path segment.")
+        candidate = Path(raw).name
+        if candidate != raw or candidate in {"", ".", ".."}:
+            raise ValueError(f"`{field_name}` must be a valid file or folder name.")
+        if candidate == ".git":
+            raise ValueError("`.git` cannot be created or renamed from the explorer.")
+        return candidate
+
+    @staticmethod
+    def _normalize_workspace_path_list(paths: Any, *, field_name: str) -> list[str]:
+        if not isinstance(paths, list) or not paths:
+            raise ValueError(f"`{field_name}` must be a non-empty list.")
+        normalized: list[str] = []
+        seen: set[str] = set()
+        for raw in paths:
+            item = QuestService._normalize_workspace_relative_path(
+                raw,
+                field_name=field_name,
+                allow_root=False,
+            )
+            if not item or item in seen:
+                continue
+            seen.add(item)
+            normalized.append(item)
+        if not normalized:
+            raise ValueError(f"`{field_name}` must include at least one valid path.")
+        return normalized
+
+    @staticmethod
+    def _filter_nested_workspace_paths(paths: list[str]) -> list[str]:
+        kept: list[str] = []
+        for path in paths:
+            if any(path == parent or path.startswith(f"{parent}/") for parent in kept):
+                continue
+            kept.append(path)
+        return kept
+
+    def _workspace_entry_payload(self, workspace_root: Path, path: Path) -> dict:
+        if path.is_dir():
+            return self._directory_node(
+                workspace_root,
+                path=path,
+                children=[],
+                git_status={},
+                changed_paths={},
+            )
+        payload = self._file_node(
+            workspace_root,
+            path=path,
+            git_status={},
+            changed_paths={},
+        )
+        if payload is None:
+            raise FileNotFoundError(f"Unknown workspace entry `{path}`.")
+        return payload
+
+    def create_workspace_folder(
+        self,
+        quest_id: str,
+        *,
+        name: str | None,
+        parent_path: str | None = None,
+    ) -> dict:
+        workspace_root = self.active_workspace_root(self._require_initialized_quest_root(quest_id))
+        normalized_parent = self._normalize_workspace_relative_path(
+            parent_path,
+            field_name="parent_path",
+            allow_root=True,
+        )
+        folder_name = self._normalize_workspace_entry_name(name, field_name="name")
+        parent = resolve_within(workspace_root, normalized_parent) if normalized_parent else workspace_root
+        if not parent.exists() or not parent.is_dir():
+            raise FileNotFoundError(
+                f"Unknown destination folder `{normalized_parent or '.'}`."
+            )
+        target = resolve_within(parent, folder_name)
+        if target.exists():
+            raise FileExistsError(
+                f"`{target.relative_to(workspace_root).as_posix()}` already exists."
+            )
+        ensure_dir(target)
+        return {
+            "ok": True,
+            "quest_id": quest_id,
+            "parent_path": normalized_parent,
+            "item": self._workspace_entry_payload(workspace_root, target),
+            "saved_at": utc_now(),
+        }
+
+    def upload_workspace_file(
+        self,
+        quest_id: str,
+        *,
+        file_name: str | None,
+        content: bytes,
+        mime_type: str | None = None,
+        parent_path: str | None = None,
+    ) -> dict:
+        workspace_root = self.active_workspace_root(self._require_initialized_quest_root(quest_id))
+        normalized_parent = self._normalize_workspace_relative_path(
+            parent_path,
+            field_name="parent_path",
+            allow_root=True,
+        )
+        safe_name = self._normalize_workspace_entry_name(file_name, field_name="file_name")
+        parent = resolve_within(workspace_root, normalized_parent) if normalized_parent else workspace_root
+        if not parent.exists() or not parent.is_dir():
+            raise FileNotFoundError(
+                f"Unknown destination folder `{normalized_parent or '.'}`."
+            )
+        target = resolve_within(parent, safe_name)
+        if target.exists():
+            raise FileExistsError(
+                f"`{target.relative_to(workspace_root).as_posix()}` already exists."
+            )
+        ensure_dir(target.parent)
+        target.write_bytes(content)
+        payload = self._workspace_entry_payload(workspace_root, target)
+        guessed_mime = mimetypes.guess_type(target.name)[0] or mime_type or "application/octet-stream"
+        payload["mime_type"] = guessed_mime
+        return {
+            "ok": True,
+            "quest_id": quest_id,
+            "parent_path": normalized_parent,
+            "item": payload,
+            "saved_at": utc_now(),
+        }
+
+    def rename_workspace_entry(
+        self,
+        quest_id: str,
+        *,
+        path: str | None,
+        new_name: str | None,
+    ) -> dict:
+        workspace_root = self.active_workspace_root(self._require_initialized_quest_root(quest_id))
+        normalized_path = self._normalize_workspace_relative_path(
+            path,
+            field_name="path",
+            allow_root=False,
+        )
+        source = resolve_within(workspace_root, normalized_path)
+        if not source.exists():
+            raise FileNotFoundError(f"Unknown workspace entry `{normalized_path}`.")
+        safe_name = self._normalize_workspace_entry_name(new_name, field_name="new_name")
+        target = resolve_within(source.parent, safe_name)
+        if target.exists() and target != source:
+            raise FileExistsError(
+                f"`{target.relative_to(workspace_root).as_posix()}` already exists."
+            )
+        if target != source:
+            source.rename(target)
+        payload = self._workspace_entry_payload(workspace_root, target)
+        return {
+            "ok": True,
+            "quest_id": quest_id,
+            "previous_path": normalized_path,
+            "item": payload,
+            "saved_at": utc_now(),
+        }
+
+    def move_workspace_entries(
+        self,
+        quest_id: str,
+        *,
+        paths: Any,
+        target_parent_path: str | None = None,
+    ) -> dict:
+        workspace_root = self.active_workspace_root(self._require_initialized_quest_root(quest_id))
+        normalized_paths = self._filter_nested_workspace_paths(
+            self._normalize_workspace_path_list(paths, field_name="paths")
+        )
+        normalized_target_parent = self._normalize_workspace_relative_path(
+            target_parent_path,
+            field_name="target_parent_path",
+            allow_root=True,
+        )
+        target_parent = (
+            resolve_within(workspace_root, normalized_target_parent)
+            if normalized_target_parent
+            else workspace_root
+        )
+        if not target_parent.exists() or not target_parent.is_dir():
+            raise FileNotFoundError(
+                f"Unknown destination folder `{normalized_target_parent or '.'}`."
+            )
+
+        moves: list[tuple[str, Path, Path]] = []
+        destination_keys: set[str] = set()
+        target_parent_resolved = target_parent.resolve()
+        for normalized_path in normalized_paths:
+            source = resolve_within(workspace_root, normalized_path)
+            if not source.exists():
+                raise FileNotFoundError(f"Unknown workspace entry `{normalized_path}`.")
+            source_resolved = source.resolve()
+            if source_resolved == target_parent_resolved or source_resolved in target_parent_resolved.parents:
+                raise ValueError(
+                    f"`{normalized_path}` cannot be moved into itself or one of its descendants."
+                )
+            destination = resolve_within(target_parent, source.name)
+            if destination.exists() and destination.resolve() != source_resolved:
+                raise FileExistsError(
+                    f"`{destination.relative_to(workspace_root).as_posix()}` already exists."
+                )
+            destination_key = str(destination.resolve())
+            if destination_key in destination_keys and destination != source:
+                raise FileExistsError(
+                    f"`{destination.relative_to(workspace_root).as_posix()}` would conflict with another moved entry."
+                )
+            destination_keys.add(destination_key)
+            moves.append((normalized_path, source, destination))
+
+        items: list[dict] = []
+        for _normalized_path, source, destination in moves:
+            if destination != source:
+                source.rename(destination)
+            items.append(self._workspace_entry_payload(workspace_root, destination))
+        return {
+            "ok": True,
+            "quest_id": quest_id,
+            "target_parent_path": normalized_target_parent,
+            "items": items,
+            "saved_at": utc_now(),
+        }
+
+    def delete_workspace_entries(
+        self,
+        quest_id: str,
+        *,
+        paths: Any,
+    ) -> dict:
+        workspace_root = self.active_workspace_root(self._require_initialized_quest_root(quest_id))
+        normalized_paths = self._filter_nested_workspace_paths(
+            self._normalize_workspace_path_list(paths, field_name="paths")
+        )
+        sources: list[Path] = []
+        items: list[dict] = []
+        for normalized_path in normalized_paths:
+            source = resolve_within(workspace_root, normalized_path)
+            if not source.exists():
+                raise FileNotFoundError(f"Unknown workspace entry `{normalized_path}`.")
+            sources.append(source)
+            items.append(self._workspace_entry_payload(workspace_root, source))
+
+        for source in sorted(sources, key=lambda item: len(item.parts), reverse=True):
+            if source.is_dir():
+                shutil.rmtree(source)
+            else:
+                source.unlink()
+        return {
+            "ok": True,
+            "quest_id": quest_id,
+            "items": items,
+            "saved_at": utc_now(),
+        }
+
     def _revision_explorer(self, quest_id: str, *, revision: str, mode: str) -> dict:
         quest_root = self._quest_root(quest_id)
         if not self._git_revision_exists(quest_root, revision):
@@ -4697,6 +5336,8 @@ class QuestService:
         }
 
     def _initialize_runtime_files(self, quest_root: Path) -> None:
+        if not self._quest_yaml_path(quest_root).exists():
+            raise FileNotFoundError(f"Unknown quest `{quest_root.name}`.")
         queue_path = self._message_queue_path(quest_root)
         if not queue_path.exists():
             write_json(queue_path, self._default_message_queue())
@@ -4828,10 +5469,11 @@ class QuestService:
             if continuation_anchor is not _UNSET:
                 normalized_anchor = str(continuation_anchor or "").strip() or None
                 if normalized_anchor is not None:
-                    from ..prompts.builder import STANDARD_SKILLS
+                    from ..prompts.builder import current_standard_skills
 
-                    if normalized_anchor not in STANDARD_SKILLS:
-                        allowed = ", ".join(STANDARD_SKILLS)
+                    available_stage_skills = current_standard_skills(repo_root())
+                    if normalized_anchor not in available_stage_skills:
+                        allowed = ", ".join(available_stage_skills)
                         raise ValueError(
                             f"Unsupported continuation anchor `{normalized_anchor}`. Allowed values: {allowed}."
                         )
@@ -5132,6 +5774,7 @@ class QuestService:
         connector_hints: dict[str, Any] | None = None,
         created_at: str | None = None,
         counts_as_visible: bool = True,
+        deliver_to_bound_conversations: bool | None = None,
     ) -> dict[str, Any]:
         timestamp = created_at or utc_now()
         payload = {
@@ -5148,6 +5791,11 @@ class QuestService:
             "reply_mode": reply_mode,
             "surface_actions": [dict(item) for item in (surface_actions or []) if isinstance(item, dict)],
             "connector_hints": dict(connector_hints) if isinstance(connector_hints, dict) else {},
+            "deliver_to_bound_conversations": (
+                bool(deliver_to_bound_conversations)
+                if deliver_to_bound_conversations is not None
+                else None
+            ),
             "created_at": timestamp,
         }
         append_jsonl(self._interaction_journal_path(quest_root), payload)
@@ -5397,6 +6045,12 @@ class QuestService:
             "queued_message_count_after_delivery": len(queue_payload.get("pending") or []),
         }
 
+    @staticmethod
+    def _document_resolution_root(quest_root: Path, workspace_root: Path, document_id: str) -> Path:
+        if document_id.startswith(("questpath::", "memory::")):
+            return quest_root
+        return workspace_root
+
     @staticmethod
     def _resolve_document(quest_root: Path, document_id: str) -> tuple[Path, bool, str, str]:
         if document_id.startswith("memory::"):
@@ -5583,12 +6237,10 @@ class QuestService:
 
     @staticmethod
     def _read_git_bytes(quest_root: Path, revision: str, relative: str) -> bytes:
-        result = subprocess.run(
+        result = run_command_bytes(
             ["git", "show", f"{revision}:{relative}"],
-            cwd=str(quest_root),
+            cwd=quest_root,
             check=False,
-            text=False,
-            capture_output=True,
         )
         if result.returncode != 0:
             raise FileNotFoundError(f"File `{relative}` does not exist at `{revision}`.")