@researai/deepscientist 1.5.13 → 1.5.15

package/src/deepscientist/quest/stage_views.py

@@ -62,6 +62,26 @@ def _field(label: str, value: object, *, tone: str = "default") -> dict[str, Any
     }
 
 
+def _selection_score_summary(value: object) -> str | None:
+    if not isinstance(value, dict):
+        return None
+    parts: list[str] = []
+    for key, raw in value.items():
+        name = str(key or "").strip()
+        if not name:
+            continue
+        if isinstance(raw, float):
+            rendered = f"{raw:.4f}".rstrip("0").rstrip(".")
+        else:
+            rendered = str(raw).strip()
+        if not rendered:
+            continue
+        parts.append(f"{name}={rendered}")
+        if len(parts) >= 4:
+            break
+    return " · ".join(parts) or None
+
+
 def _evaluation_summary(value: object) -> dict[str, Any]:
     if not isinstance(value, dict):
         return {}
@@ -215,6 +235,8 @@ class QuestStageViewBuilder:
     def build(self) -> dict[str, Any]:
         selection_type = str(self.selection.get("selection_type") or "").strip()
         self.stage_key = self._resolve_effective_stage_key()
+        if selection_type == "idea_candidate":
+            return self._build_idea_candidate()
         if selection_type == "branch_node" and self.stage_key not in {"experiment", "analysis", "paper"}:
             return self._build_branch()
         if self.stage_key == "baseline":
@@ -280,6 +302,8 @@ class QuestStageViewBuilder:
             normalized = [str(item).strip() for item in raw if str(item).strip()]
             if normalized:
                 return normalized
+        if str(self.selection.get("selection_type") or "").strip() == "idea_candidate":
+            return self._idea_candidate_scope_paths()
         if str(self.selection.get("selection_type") or "").strip() == "branch_node":
             return self._branch_scope_paths()
         defaults = {
@@ -344,37 +368,83 @@ class QuestStageViewBuilder:
             return True
         return False
 
-    def _path_in_quest(self, raw_path: object) -> tuple[Path, str] | None:
+    def _path_in_quest(self, raw_path: object) -> tuple[Path, str, str] | None:
         text = str(raw_path or "").strip()
         if not text:
             return None
         path = Path(text)
+        candidates: list[Path] = []
         if not path.is_absolute():
-            path = (self.quest_root / text).resolve()
+            for base in (self.workspace_root, self.quest_root):
+                try:
+                    candidates.append((base / text).resolve())
+                except OSError:
+                    continue
         else:
             try:
-                path = path.resolve()
+                candidates.append(path.resolve())
             except OSError:
                 return None
-        try:
-            relative = path.relative_to(self.quest_root.resolve()).as_posix()
-        except ValueError:
+
+        if not candidates:
             return None
-        return path, relative
+
+        seen: set[str] = set()
+        unique_candidates: list[Path] = []
+        for candidate in candidates:
+            key = str(candidate)
+            if key in seen:
+                continue
+            seen.add(key)
+            unique_candidates.append(candidate)
+
+        existing_candidates: list[Path] = []
+        missing_candidates: list[Path] = []
+        for candidate in unique_candidates:
+            try:
+                if candidate.exists():
+                    existing_candidates.append(candidate)
+                else:
+                    missing_candidates.append(candidate)
+            except OSError:
+                missing_candidates.append(candidate)
+
+        ordered = [*existing_candidates, *missing_candidates]
+
+        workspace_root = self.workspace_root.resolve()
+        quest_root = self.quest_root.resolve()
+        for candidate in ordered:
+            if candidate.exists():
+                try:
+                    relative = candidate.relative_to(workspace_root).as_posix()
+                    return candidate, relative, "path"
+                except ValueError:
+                    pass
+            try:
+                relative = candidate.relative_to(quest_root).as_posix()
+                return candidate, relative, "questpath"
+            except ValueError:
+                pass
+            try:
+                relative = candidate.relative_to(workspace_root).as_posix()
+                return candidate, relative, "path"
+            except ValueError:
+                continue
+        return None
 
     def _document_id_for_path(self, raw_path: object) -> str | None:
         resolved = self._path_in_quest(raw_path)
         if resolved is None:
             return None
-        path, relative = resolved
+        path, relative, document_scope = resolved
         if path.exists() and path.is_file():
-            return f"questpath::{relative}"
+            return f"{document_scope}::{relative}"
         return None
 
     def _relative_path_or_raw(self, raw_path: object) -> str | None:
         resolved = self._path_in_quest(raw_path)
         if resolved is not None:
-            _path, relative = resolved
+            _path, relative, _document_scope = resolved
             return relative
         text = str(raw_path or "").strip()
         return text or None
@@ -383,7 +453,7 @@ class QuestStageViewBuilder:
         resolved = self._path_in_quest(raw_path)
         if resolved is None:
             return None
-        path, _relative = resolved
+        path, _relative, _document_scope = resolved
         if not path.exists() or not path.is_file():
             return None
         try:
@@ -464,7 +534,7 @@ class QuestStageViewBuilder:
                 "exists": path.exists(),
                 "scope": "external",
             }
-        path, relative = resolved
+        path, relative, document_scope = resolved
         exists = path.exists()
         kind = "directory" if (exists and path.is_dir()) or expected_kind == "directory" else "file"
         scope = self.quest_service._classify_relative_scope(relative)[0]
@@ -474,7 +544,7 @@ class QuestStageViewBuilder:
             "description": description,
             "path": relative,
             "absolute_path": str(path),
-            "document_id": f"questpath::{relative}" if exists and path.is_file() else None,
+            "document_id": f"{document_scope}::{relative}" if exists and path.is_file() else None,
             "kind": kind,
             "exists": exists,
             "scope": scope,
@@ -508,30 +578,88 @@ class QuestStageViewBuilder:
         resolved = self._path_in_quest(raw_path)
         if resolved is None:
             return None
-        _path, relative = resolved
+        _path, relative, _document_scope = resolved
         return relative
 
-    def _paper_latex_root(self, bundle_manifest: dict[str, Any]) -> str | None:
-        preferred = self._paper_relative_path(bundle_manifest.get("latex_root_path"))
-        if preferred:
-            return preferred
+    def _paper_latex_root(
+        self,
+        bundle_manifest: dict[str, Any],
+        *,
+        compile_report: dict[str, Any] | None = None,
+    ) -> str | None:
+        for candidate in (
+            bundle_manifest.get("latex_root_path"),
+            (compile_report or {}).get("latex_root_path"),
+            (compile_report or {}).get("main_file_path"),
+        ):
+            resolved = self._path_in_quest(candidate)
+            if resolved is None:
+                continue
+            path, relative, _document_scope = resolved
+            if path.is_dir():
+                return relative
+            if path.suffix.lower() == ".tex":
+                return PurePosixPath(relative).parent.as_posix()
         paper_root = self._paper_root()
         for candidate in (paper_root / "latex", paper_root / "tex"):
             if candidate.exists():
-                return candidate.relative_to(self.quest_root).as_posix()
+                try:
+                    return candidate.relative_to(self.workspace_root.resolve()).as_posix()
+                except ValueError:
+                    return candidate.relative_to(self.quest_root).as_posix()
         return None
 
-    def _paper_main_tex(self, latex_root_rel: str | None) -> str | None:
+    def _paper_main_tex(
+        self,
+        latex_root_rel: str | None,
+        *,
+        bundle_manifest: dict[str, Any] | None = None,
+        compile_report: dict[str, Any] | None = None,
+    ) -> str | None:
+        for candidate in (
+            (compile_report or {}).get("main_file_path"),
+            bundle_manifest.get("main_tex_path") if isinstance(bundle_manifest, dict) else None,
+            bundle_manifest.get("latex_root_path") if isinstance(bundle_manifest, dict) else None,
+            (compile_report or {}).get("latex_root_path"),
+        ):
+            resolved = self._path_in_quest(candidate)
+            if resolved is None:
+                continue
+            path, relative, _document_scope = resolved
+            if path.suffix.lower() == ".tex":
+                return relative
+            if path.is_dir():
+                preferred = path / "main.tex"
+                if preferred.exists():
+                    nested = self._path_in_quest(preferred)
+                    if nested is not None:
+                        _resolved_path, nested_relative, _nested_scope = nested
+                        return nested_relative
         if not latex_root_rel:
             return None
-        latex_root = self.quest_root / latex_root_rel
+        latex_root = (self.workspace_root / latex_root_rel).resolve()
+        if not latex_root.exists():
+            latex_root = (self.quest_root / latex_root_rel).resolve()
+        if latex_root.is_file() and latex_root.suffix.lower() == ".tex":
+            nested = self._path_in_quest(latex_root)
+            if nested is not None:
+                _resolved_path, nested_relative, _nested_scope = nested
+                return nested_relative
+            return None
         preferred = latex_root / "main.tex"
         if preferred.exists():
-            return preferred.relative_to(self.quest_root).as_posix()
+            nested = self._path_in_quest(preferred)
+            if nested is not None:
+                _resolved_path, nested_relative, _nested_scope = nested
+                return nested_relative
         candidates = sorted(latex_root.glob("*.tex"))
         if not candidates:
             return None
-        return candidates[0].relative_to(self.quest_root).as_posix()
+        nested = self._path_in_quest(candidates[0])
+        if nested is None:
+            return None
+        _resolved_path, nested_relative, _nested_scope = nested
+        return nested_relative
 
     def _paper_pdf_candidates(
         self,
@@ -589,6 +717,13 @@ class QuestStageViewBuilder:
             "artifacts/reports",
         ]
 
+    def _idea_candidate_scope_paths(self) -> list[str]:
+        candidate_id = str(self.selection.get("selection_ref") or self.selection.get("idea_id") or "").strip()
+        return [
+            *( [f"memory/ideas/_candidates/{candidate_id}"] if candidate_id else []),
+            "artifacts/reports",
+        ]
+
     def _experiment_scope_paths(self, run_id: str | None) -> list[str]:
         return [
             *( [f"experiments/main/{run_id}"] if run_id else []),
@@ -813,6 +948,25 @@ class QuestStageViewBuilder:
                     items.append(item)
         return items
 
+    def _idea_candidate_stage_items(self) -> list[dict[str, Any]]:
+        candidate_id = str(self.selection.get("selection_ref") or self.selection.get("idea_id") or "").strip()
+        if not candidate_id:
+            return []
+        items: list[dict[str, Any]] = []
+        for item in self.artifacts:
+            payload = self._payload(item)
+            if str(payload.get("kind") or "").strip() != "idea":
+                continue
+            if str(payload.get("idea_id") or "").strip() != candidate_id:
+                continue
+            flow_type = str(payload.get("flow_type") or "").strip()
+            protocol_step = str(payload.get("protocol_step") or "").strip()
+            details = dict(payload.get("details") or {}) if isinstance(payload.get("details"), dict) else {}
+            submission_mode = str(details.get("submission_mode") or payload.get("submission_mode") or "").strip().lower()
+            if flow_type == "idea_submission" and (protocol_step == "candidate" or submission_mode == "candidate"):
+                items.append(item)
+        return items
+
     def _build_idea(self) -> dict[str, Any]:
         idea_items = self._idea_stage_items()
         latest = idea_items[-1] if idea_items else None
@@ -840,6 +994,8 @@ class QuestStageViewBuilder:
         draft_md_rel_path = self._relative_path_or_raw(draft_md_path)
         draft_markdown = self._markdown_body_for_path(draft_md_path)
         lineage_intent = str(payload.get("lineage_intent") or details.get("lineage_intent") or "").strip() or None
+        selection_scores = details.get("selection_scores")
+        selection_score_summary = _selection_score_summary(selection_scores)
         note = (
             str(payload.get("summary") or payload.get("reason") or "").strip()
             or "No durable idea submission has been recorded yet."
@@ -883,6 +1039,11 @@ class QuestStageViewBuilder:
                 _field("Problem", details.get("problem") or "Not recorded"),
                 _field("Hypothesis", details.get("hypothesis") or "Not recorded"),
                 _field("Mechanism", details.get("mechanism") or "Not recorded"),
+                _field("Method Brief", details.get("method_brief") or "Not recorded"),
+                _field("Selection Scores", selection_score_summary or "Not recorded"),
+                _field("Mechanism Family", details.get("mechanism_family") or "Not recorded"),
+                _field("Change Layer", details.get("change_layer") or "Not recorded"),
+                _field("Source Lens", details.get("source_lens") or "Not recorded"),
                 _field("Expected Gain", details.get("expected_gain") or "Not recorded"),
                 _field("Risks", details.get("risks") or "Not recorded"),
                 _field("Evidence Paths", details.get("evidence_paths") or "Not recorded"),
@@ -905,6 +1066,11 @@ class QuestStageViewBuilder:
                     "problem": details.get("problem"),
                     "hypothesis": details.get("hypothesis"),
                     "mechanism": details.get("mechanism"),
+                    "method_brief": details.get("method_brief"),
+                    "selection_scores": selection_scores or None,
+                    "mechanism_family": details.get("mechanism_family"),
+                    "change_layer": details.get("change_layer"),
+                    "source_lens": details.get("source_lens"),
                     "expected_gain": details.get("expected_gain"),
                     "risks": details.get("risks") or [],
                     "evidence_paths": details.get("evidence_paths") or [],
@@ -924,6 +1090,101 @@ class QuestStageViewBuilder:
             subviews=["overview", "details", "draft"] if draft_markdown else ["overview", "details"],
         )
 
+    def _build_idea_candidate(self) -> dict[str, Any]:
+        candidate_items = self._idea_candidate_stage_items()
+        latest = candidate_items[-1] if candidate_items else None
+        payload = self._payload(latest or {})
+        details = dict(payload.get("details") or {}) if isinstance(payload.get("details"), dict) else {}
+        candidate_id = str(self.selection.get("selection_ref") or payload.get("idea_id") or "candidate").strip() or "candidate"
+        title_text = (
+            str(details.get("title") or self.selection.get("label") or candidate_id).strip() or candidate_id
+        )
+        paths = dict(payload.get("paths") or {}) if isinstance(payload.get("paths"), dict) else {}
+        candidate_root = paths.get("candidate_root") or str(self.quest_root / "memory" / "ideas" / "_candidates" / candidate_id)
+        idea_md_path = paths.get("idea_md") or str(Path(candidate_root) / "idea.md")
+        draft_md_path = paths.get("idea_draft_md") or details.get("idea_draft_path") or str(Path(candidate_root) / "draft.md")
+        idea_markdown = self._markdown_body_for_path(idea_md_path)
+        draft_markdown = self._markdown_body_for_path(draft_md_path)
+        idea_md_rel_path = self._relative_path_or_raw(idea_md_path)
+        draft_md_rel_path = self._relative_path_or_raw(draft_md_path)
+        candidate_root_rel_path = self._relative_path_or_raw(candidate_root)
+        selection_scores = details.get("selection_scores")
+        selection_score_summary = _selection_score_summary(selection_scores)
+        note = (
+            str(payload.get("summary") or payload.get("reason") or self.selection.get("summary") or "").strip()
+            or "No durable candidate brief summary has been recorded yet."
+        )
+        lineage_intent = str(payload.get("lineage_intent") or details.get("lineage_intent") or "").strip() or None
+        parent_branch = str(payload.get("parent_branch") or details.get("parent_branch") or self.selection.get("branch_name") or "").strip() or None
+        foundation_reason = str(payload.get("foundation_reason") or details.get("foundation_reason") or "").strip() or None
+        return self._base_payload(
+            title=f"Candidate Brief · {title_text}",
+            note=note,
+            status=str(payload.get("status") or "candidate").strip() or "candidate",
+            tags=[
+                "candidate-brief",
+                details.get("mechanism_family") or "",
+                details.get("change_layer") or "",
+                details.get("source_lens") or "",
+                lineage_intent or "",
+            ],
+            overview=[
+                _field("Candidate ID", candidate_id),
+                _field("Parent Branch", parent_branch or "Not recorded"),
+                _field("Next Target", details.get("next_target") or "optimize"),
+                _field("Candidate Root", candidate_root_rel_path or candidate_root),
+            ],
+            key_facts=[
+                _field("Problem", details.get("problem") or "Not recorded"),
+                _field("Hypothesis", details.get("hypothesis") or "Not recorded"),
+                _field("Mechanism", details.get("mechanism") or "Not recorded"),
+                _field("Method Brief", details.get("method_brief") or "Not recorded"),
+                _field("Selection Scores", selection_score_summary or "Not recorded"),
+                _field("Mechanism Family", details.get("mechanism_family") or "Not recorded"),
+                _field("Change Layer", details.get("change_layer") or "Not recorded"),
+                _field("Source Lens", details.get("source_lens") or "Not recorded"),
+                _field("Expected Gain", details.get("expected_gain") or "Not recorded"),
+                _field("Foundation Reason", foundation_reason or "Not recorded"),
+            ],
+            key_files=self._dedupe_files(
+                [
+                    self._file_entry(candidate_root, label="Candidate Root", description="Branchless candidate brief workspace.", expected_kind="directory"),
+                    self._file_entry(idea_md_path, label="Candidate Markdown", description="Durable candidate brief document."),
+                    self._file_entry(draft_md_path, label="Candidate Draft", description="Long-form candidate brief draft."),
+                ]
+            ),
+            history=self._artifact_history(candidate_items),
+            details={
+                "idea": {
+                    "idea_id": candidate_id,
+                    "title": title_text,
+                    "problem": details.get("problem"),
+                    "hypothesis": details.get("hypothesis"),
+                    "mechanism": details.get("mechanism"),
+                    "method_brief": details.get("method_brief"),
+                    "selection_scores": selection_scores or None,
+                    "mechanism_family": details.get("mechanism_family"),
+                    "change_layer": details.get("change_layer"),
+                    "source_lens": details.get("source_lens"),
+                    "expected_gain": details.get("expected_gain"),
+                    "next_target": details.get("next_target") or "optimize",
+                    "lineage_intent": lineage_intent,
+                    "parent_branch": parent_branch,
+                    "candidate_root": candidate_root_rel_path or candidate_root,
+                    "idea_path": idea_md_rel_path,
+                    "idea_markdown": idea_markdown,
+                    "draft_path": draft_md_rel_path,
+                    "draft_markdown": draft_markdown,
+                    "decision_reason": payload.get("reason"),
+                },
+                "latest_artifact": self._artifact_detail(latest, payload),
+            },
+            lineage_intent=lineage_intent,
+            idea_draft_path=draft_md_rel_path,
+            draft_available=bool(draft_markdown),
+            subviews=["overview", "details", "draft"] if draft_markdown else ["overview", "details"],
+        )
+
     def _build_branch(self) -> dict[str, Any]:
         idea_items = [
             item
@@ -946,6 +1207,8 @@ class QuestStageViewBuilder:
         idea_title = str(latest_idea_details.get("title") or "").strip() or None
         idea_problem = str(latest_idea_details.get("problem") or "").strip() or None
         next_target = str(latest_idea_details.get("next_target") or "").strip() or None
+        selection_scores = latest_idea_details.get("selection_scores")
+        selection_score_summary = _selection_score_summary(selection_scores)
         lineage_intent = str(
             latest_idea_payload.get("lineage_intent")
             or latest_idea_details.get("lineage_intent")
@@ -1065,6 +1328,11 @@ class QuestStageViewBuilder:
             key_facts=[
                 _field("Idea Title", idea_title or "Not recorded"),
                 _field("Idea Problem", idea_problem or "Not recorded"),
+                _field("Method Brief", latest_idea_details.get("method_brief") or "Not recorded"),
+                _field("Selection Scores", selection_score_summary or "Not recorded"),
+                _field("Mechanism Family", latest_idea_details.get("mechanism_family") or "Not recorded"),
+                _field("Change Layer", latest_idea_details.get("change_layer") or "Not recorded"),
+                _field("Source Lens", latest_idea_details.get("source_lens") or "Not recorded"),
                 _field("Foundation", foundation_label or "Current head"),
                 _field("Foundation Reason", foundation_reason or "Not recorded"),
                 _field("Next Target", next_target or "Not recorded"),
@@ -1120,6 +1388,11 @@ class QuestStageViewBuilder:
                     "lineage_intent": lineage_intent,
                     "idea_title": idea_title,
                     "idea_problem": idea_problem,
+                    "method_brief": latest_idea_details.get("method_brief"),
+                    "selection_scores": selection_scores or None,
+                    "mechanism_family": latest_idea_details.get("mechanism_family"),
+                    "change_layer": latest_idea_details.get("change_layer"),
+                    "source_lens": latest_idea_details.get("source_lens"),
                     "next_target": next_target,
                     "idea_draft_path": idea_draft_rel_path,
                     "idea_draft_markdown": idea_draft_markdown,
@@ -1460,10 +1733,11 @@ class QuestStageViewBuilder:
             },
         )
 
-    def _paper_files(self) -> list[dict[str, Any]]:
+    def _paper_files(self, *, compile_report: dict[str, Any] | None = None) -> list[dict[str, Any]]:
         bundle_manifest = self._paper_bundle_manifest()
-        latex_root_rel = self._paper_latex_root(bundle_manifest)
-        main_tex_rel = self._paper_main_tex(latex_root_rel)
+        compile_report = compile_report if isinstance(compile_report, dict) else {}
+        latex_root_rel = self._paper_latex_root(bundle_manifest, compile_report=compile_report)
+        main_tex_rel = self._paper_main_tex(latex_root_rel, bundle_manifest=bundle_manifest, compile_report=compile_report)
         pdf_candidates = self._paper_pdf_candidates(bundle_manifest, main_tex_rel=main_tex_rel)
         paper_root = self._paper_root()
         open_source_root = self._open_source_root()
@@ -1474,11 +1748,13 @@ class QuestStageViewBuilder:
                 for path in candidates
             ],
             self._file_entry(paper_root / "selected_outline.json", label="Selected Outline", description="Chosen paper outline."),
+            self._file_entry(paper_root / "outline" / "manifest.json", label="Outline Manifest", description="Author-facing paper outline manifest."),
             self._file_entry(paper_root / "outline_selection.md", label="Outline Selection Note", description="Outline selection rationale."),
             self._file_entry(paper_root / "draft.md", label="Draft Markdown", description="Current paper draft."),
             self._file_entry(paper_root / "writing_plan.md", label="Writing Plan", description="Paper writing plan."),
             self._file_entry(paper_root / "references.bib", label="References", description="Bibliography file."),
             self._file_entry(paper_root / "claim_evidence_map.json", label="Claim-Evidence Map", description="Claim to evidence mapping."),
+            self._file_entry(paper_root / "paper_line_state.json", label="Paper Line State", description="Derived summary state for the active paper line."),
             self._file_entry(paper_root / "baseline_inventory.json", label="Baseline Inventory", description="Canonical and supplementary baseline inventory for writing."),
             self._file_entry(paper_root / "build" / "compile_report.json", label="Compile Report", description="Paper build/compile report."),
             self._file_entry(paper_root / "paper_bundle_manifest.json", label="Bundle Manifest", description="Final paper bundle manifest."),
@@ -1537,8 +1813,8 @@ class QuestStageViewBuilder:
         if not isinstance(compile_report, dict):
             compile_report = {}
         bundle_manifest = self._paper_bundle_manifest()
-        latex_root_rel = self._paper_latex_root(bundle_manifest)
-        main_tex_rel = self._paper_main_tex(latex_root_rel)
+        latex_root_rel = self._paper_latex_root(bundle_manifest, compile_report=compile_report)
+        main_tex_rel = self._paper_main_tex(latex_root_rel, bundle_manifest=bundle_manifest, compile_report=compile_report)
         references_bib = read_text(paper_root / "references.bib", "")
         references_count = sum(1 for line in references_bib.splitlines() if line.lstrip().startswith("@"))
         pdf_paths = self._paper_pdf_candidates(bundle_manifest, main_tex_rel=main_tex_rel)
@@ -1577,7 +1853,7 @@ class QuestStageViewBuilder:
                 _field("LaTeX Root", latex_root_rel or "Not recorded"),
                 _field("Main TeX", main_tex_rel or "Not recorded"),
             ],
-            key_files=self._paper_files(),
+            key_files=self._paper_files(compile_report=compile_report),
             history=self._artifact_history(paper_items),
             details={
                 "paper": {

package/src/deepscientist/runners/base.py

@@ -17,6 +17,8 @@ class RunRequest:
     approval_policy: str
     sandbox_mode: str
     turn_reason: str = "user_message"
+    turn_intent: str = "continue_stage"
+    turn_mode: str = "stage_execution"
     reasoning_effort: str | None = None
     turn_id: str | None = None
     attempt_index: int = 1