@researai/deepscientist 1.5.7 → 1.5.8

package/src/deepscientist/quest/service.py

@@ -17,7 +17,7 @@ try:
 except ImportError:  # pragma: no cover
     fcntl = None
 
-from ..artifact.metrics import build_metrics_timeline, normalize_metrics_summary
+from ..artifact.metrics import build_metrics_timeline, extract_latest_metric
 from ..config import ConfigManager
 from ..connector_runtime import conversation_identity_key, normalize_conversation_id
 from ..gitops import current_branch, export_git_graph, head_commit, init_repo
@@ -129,6 +129,9 @@ class QuestService:
             "active_analysis_campaign_id": None,
             "analysis_parent_branch": None,
             "analysis_parent_worktree_root": None,
+            "paper_parent_branch": None,
+            "paper_parent_worktree_root": None,
+            "paper_parent_run_id": None,
             "next_pending_slice_id": None,
             "workspace_mode": "quest",
             "last_flow_type": None,
@@ -151,6 +154,9 @@ class QuestService:
         parent_root = str(merged.get("analysis_parent_worktree_root") or "").strip()
         if parent_root and not Path(parent_root).exists():
             merged["analysis_parent_worktree_root"] = None
+        paper_parent_root = str(merged.get("paper_parent_worktree_root") or "").strip()
+        if paper_parent_root and not Path(paper_parent_root).exists():
+            merged["paper_parent_worktree_root"] = None
         return merged
 
     def write_research_state(self, quest_root: Path, payload: dict[str, Any]) -> dict[str, Any]:
@@ -206,9 +212,37 @@ class QuestService:
     def _artifact_roots(self, quest_root: Path) -> list[Path]:
         return [root for root in self.workspace_roots(quest_root) if (root / "artifacts").exists()]
 
+    @staticmethod
+    def _artifact_item_identity(path: Path, payload: dict[str, Any], *, kind: str) -> str:
+        normalized_kind = str(kind or payload.get("kind") or path.parent.name or "artifact").strip() or "artifact"
+        artifact_id = str(payload.get("artifact_id") or payload.get("id") or "").strip()
+        if artifact_id:
+            return f"{normalized_kind}:artifact:{artifact_id}"
+        branch_name = str(payload.get("branch") or "").strip()
+        run_id = str(payload.get("run_id") or "").strip()
+        if normalized_kind == "runs" and run_id and branch_name:
+            return f"{normalized_kind}:branch_run:{branch_name}:{run_id}"
+        if normalized_kind == "runs" and run_id:
+            return f"{normalized_kind}:run:{run_id}"
+        idea_id = str(payload.get("idea_id") or "").strip()
+        if normalized_kind == "ideas" and idea_id and branch_name:
+            return f"{normalized_kind}:branch_idea:{branch_name}:{idea_id}"
+        if normalized_kind == "ideas" and idea_id:
+            return f"{normalized_kind}:idea:{idea_id}"
+        return f"path:{path.resolve()}"
+
+    @staticmethod
+    def _artifact_item_rank(payload: dict[str, Any], *, path: Path, mtime_ns: int) -> tuple[str, str, int, int, str]:
+        return (
+            str(payload.get("updated_at") or ""),
+            str(payload.get("created_at") or ""),
+            len(payload),
+            mtime_ns,
+            str(path),
+        )
+
     def _collect_artifacts(self, quest_root: Path) -> list[dict[str, Any]]:
-        artifacts: list[dict[str, Any]] = []
-        seen_paths: set[str] = set()
+        artifacts_by_identity: dict[str, dict[str, Any]] = {}
         for root in self._artifact_roots(quest_root):
             artifacts_root = root / "artifacts"
             if not artifacts_root.exists():
@@ -217,19 +251,37 @@ class QuestService:
                 if not folder.is_dir():
                     continue
                 for path in sorted(folder.glob("*.json")):
-                    resolved_key = str(path.resolve())
-                    if resolved_key in seen_paths:
-                        continue
-                    seen_paths.add(resolved_key)
                     item = self._read_cached_json(path, {})
-                    artifacts.append(
-                        {
-                            "kind": folder.name,
-                            "path": str(path),
-                            "payload": item,
-                            "workspace_root": str(root),
-                        }
-                    )
+                    payload = item if isinstance(item, dict) else {}
+                    try:
+                        mtime_ns = path.stat().st_mtime_ns
+                    except OSError:
+                        mtime_ns = 0
+                    artifact = {
+                        "kind": folder.name,
+                        "path": str(path),
+                        "payload": item,
+                        "workspace_root": str(root),
+                    }
+                    identity = self._artifact_item_identity(path, payload, kind=folder.name)
+                    existing = artifacts_by_identity.get(identity)
+                    existing_payload = existing.get("payload") if isinstance((existing or {}).get("payload"), dict) else {}
+                    existing_path = Path(str((existing or {}).get("path") or path))
+                    try:
+                        existing_mtime_ns = existing_path.stat().st_mtime_ns if existing else 0
+                    except OSError:
+                        existing_mtime_ns = 0
+                    if existing is None or self._artifact_item_rank(
+                        payload,
+                        path=path,
+                        mtime_ns=mtime_ns,
+                    ) >= self._artifact_item_rank(
+                        existing_payload,
+                        path=existing_path,
+                        mtime_ns=existing_mtime_ns,
+                    ):
+                        artifacts_by_identity[identity] = artifact
+        artifacts = list(artifacts_by_identity.values())
         artifacts.sort(
             key=lambda item: str(
                 ((item.get("payload") or {}).get("updated_at"))
@@ -267,22 +319,7 @@ class QuestService:
 
     @staticmethod
     def _latest_metric_from_payload(payload: dict[str, Any]) -> dict[str, Any] | None:
-        metrics_summary = normalize_metrics_summary(payload.get("metrics_summary"))
-        if not metrics_summary:
-            return None
-        progress_eval = payload.get("progress_eval") if isinstance(payload.get("progress_eval"), dict) else {}
-        comparisons = payload.get("baseline_comparisons") if isinstance(payload.get("baseline_comparisons"), dict) else {}
-        primary_metric_id = (
-            str(progress_eval.get("primary_metric_id") or comparisons.get("primary_metric_id") or "").strip()
-            or next(iter(metrics_summary.keys()))
-        )
-        result = {
-            "key": primary_metric_id,
-            "value": metrics_summary.get(primary_metric_id),
-        }
-        if progress_eval.get("delta_vs_baseline") is not None:
-            result["delta_vs_baseline"] = progress_eval.get("delta_vs_baseline")
-        return result
+        return extract_latest_metric(payload)
 
     @staticmethod
     def _parse_numeric_quest_id(value: str | None) -> int | None:
@@ -980,6 +1017,9 @@ class QuestService:
             "active_analysis_campaign_id": research_state.get("active_analysis_campaign_id"),
             "analysis_parent_branch": research_state.get("analysis_parent_branch"),
             "analysis_parent_worktree_root": research_state.get("analysis_parent_worktree_root"),
+            "paper_parent_branch": research_state.get("paper_parent_branch"),
+            "paper_parent_worktree_root": research_state.get("paper_parent_worktree_root"),
+            "paper_parent_run_id": research_state.get("paper_parent_run_id"),
             "next_pending_slice_id": research_state.get("next_pending_slice_id"),
             "workspace_mode": research_state.get("workspace_mode") or "quest",
             "active_baseline_id": active_baseline_id,
@@ -1981,7 +2021,11 @@ class QuestService:
                 },
             }
 
-        resolution_root = quest_root if document_id.startswith("questpath::") else workspace_root
+        resolution_root = (
+            quest_root
+            if document_id.startswith(("questpath::", "memory::"))
+            else workspace_root
+        )
         path, writable, scope, source_kind = self._resolve_document(resolution_root, document_id)
         renderer_hint, mime_type = self._renderer_hint_for(path)
         is_text = self._is_text_document(path, mime_type, renderer_hint)
@@ -2162,11 +2206,16 @@ class QuestService:
         asset_name = f"{safe_stem}-{generate_id('asset').split('-', 1)[1]}{asset_suffix}"
         asset_relative_dir = self._markdown_asset_directory(base_relative)
         asset_relative = (asset_relative_dir / asset_name).as_posix()
-        asset_root = quest_root if document_id.startswith("questpath::") else workspace_root
+        asset_root = (
+            quest_root
+            if document_id.startswith(("questpath::", "memory::"))
+            else workspace_root
+        )
         asset_path = resolve_within(asset_root, asset_relative)
         ensure_dir(asset_path.parent)
         asset_path.write_bytes(content)
-        asset_document_id = f"{'questpath' if document_id.startswith('questpath::') else 'path'}::{asset_relative}"
+        asset_document_scope = "questpath" if document_id.startswith(("questpath::", "memory::")) else "path"
+        asset_document_id = f"{asset_document_scope}::{asset_relative}"
         relative_markdown_path = self._relative_path_from_base(base_relative, asset_relative)
         return {
             "ok": True,

package/src/deepscientist/quest/stage_views.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import json
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 from typing import Any
 
 from ..memory.frontmatter import load_markdown_document
@@ -117,7 +117,7 @@ class QuestStageViewBuilder:
         self.snapshot = dict(snapshot or {})
         self.selection = dict(selection or {})
         self.trace = dict(trace or {})
-        self.workspace_root = Path(str(self.snapshot.get("active_workspace_root") or quest_root))
+        self.workspace_root = self._resolve_workspace_root()
         self.branch_name = (
             str(
                 self.selection.get("branch_name")
@@ -135,6 +135,24 @@ class QuestStageViewBuilder:
         self.stage_status = str(self.selection.get("status") or self.trace.get("status") or "").strip() or None
         self.artifacts = sorted(list(self.quest_service._collect_artifacts(quest_root)), key=_artifact_sort_key)
 
+    def _resolve_workspace_root(self) -> Path:
+        for raw in (
+            self.selection.get("worktree_rel_path"),
+            self.trace.get("worktree_rel_path"),
+            self.selection.get("worktree_root"),
+            self.trace.get("worktree_root"),
+            self.snapshot.get("active_workspace_root"),
+        ):
+            text = str(raw or "").strip()
+            if not text:
+                continue
+            candidate = Path(text)
+            if not candidate.is_absolute():
+                candidate = (self.quest_root / text).resolve()
+            if candidate.exists():
+                return candidate
+        return self.quest_root
+
     def build(self) -> dict[str, Any]:
         if str(self.selection.get("selection_type") or "").strip() == "branch_node":
             return self._build_branch()
@@ -208,7 +226,7 @@ class QuestStageViewBuilder:
             "idea": self._idea_scope_paths(),
             "experiment": self._experiment_scope_paths(None),
             "analysis": self._analysis_scope_paths(None),
-            "paper": ["paper"],
+            "paper": self._paper_scope_paths(),
         }
         return defaults.get(self.stage_key, [])
 
@@ -356,9 +374,29 @@ class QuestStageViewBuilder:
         }
 
     def _paper_bundle_manifest(self) -> dict[str, Any]:
-        payload = read_json(self.quest_root / "paper" / "paper_bundle_manifest.json", {})
+        payload = read_json(self._paper_root() / "paper_bundle_manifest.json", {})
         return payload if isinstance(payload, dict) else {}
 
+    def _paper_root(self) -> Path:
+        candidates = [self.workspace_root / "paper", self.quest_root / "paper"]
+        for candidate in candidates:
+            if candidate.exists():
+                return candidate
+        return candidates[0]
+
+    def _paper_scope_paths(self) -> list[str]:
+        try:
+            return [self._paper_root().relative_to(self.quest_root).as_posix()]
+        except ValueError:
+            return ["paper"]
+
+    def _open_source_root(self) -> Path:
+        candidates = [self.workspace_root / "release" / "open_source", self.quest_root / "release" / "open_source"]
+        for candidate in candidates:
+            if candidate.exists():
+                return candidate
+        return candidates[0]
+
     def _paper_relative_path(self, raw_path: object) -> str | None:
         resolved = self._path_in_quest(raw_path)
         if resolved is None:
@@ -370,9 +408,10 @@ class QuestStageViewBuilder:
         preferred = self._paper_relative_path(bundle_manifest.get("latex_root_path"))
         if preferred:
             return preferred
-        for candidate in ("paper/latex", "paper/tex"):
-            if (self.quest_root / candidate).exists():
-                return candidate
+        paper_root = self._paper_root()
+        for candidate in (paper_root / "latex", paper_root / "tex"):
+            if candidate.exists():
+                return candidate.relative_to(self.quest_root).as_posix()
         return None
 
     def _paper_main_tex(self, latex_root_rel: str | None) -> str | None:
@@ -401,7 +440,7 @@ class QuestStageViewBuilder:
             guessed = str(PurePosixPath(main_tex_rel).with_suffix(".pdf"))
             if (self.quest_root / guessed).exists():
                 candidates.append(guessed)
-        for path in sorted((self.quest_root / "paper").glob("*.pdf")):
+        for path in sorted(self._paper_root().glob("*.pdf")):
             candidates.append(path.relative_to(self.quest_root).as_posix())
         deduped: list[str] = []
         seen: set[str] = set()
@@ -948,7 +987,7 @@ class QuestStageViewBuilder:
                         description="Merged analysis summary for this branch.",
                     ),
                     self._file_entry(
-                        "paper/selected_outline.json" if (self.quest_root / "paper" / "selected_outline.json").exists() else None,
+                        (self._paper_root() / "selected_outline.json") if (self._paper_root() / "selected_outline.json").exists() else None,
                         label="Selected Outline",
                         description="Current selected paper outline.",
                     ),
@@ -1264,23 +1303,25 @@ class QuestStageViewBuilder:
         latex_root_rel = self._paper_latex_root(bundle_manifest)
         main_tex_rel = self._paper_main_tex(latex_root_rel)
         pdf_candidates = self._paper_pdf_candidates(bundle_manifest, main_tex_rel=main_tex_rel)
-        candidates = sorted((self.quest_root / "paper" / "outlines" / "candidates").glob("*.json"))
+        paper_root = self._paper_root()
+        open_source_root = self._open_source_root()
+        candidates = sorted((paper_root / "outlines" / "candidates").glob("*.json"))
         files: list[dict[str, Any] | None] = [
             *[
                 self._file_entry(path, label=f"Outline Candidate · {path.stem}", description="Paper outline candidate JSON.")
                 for path in candidates
             ],
-            self._file_entry("paper/selected_outline.json", label="Selected Outline", description="Chosen paper outline."),
-            self._file_entry("paper/outline_selection.md", label="Outline Selection Note", description="Outline selection rationale."),
-            self._file_entry("paper/draft.md", label="Draft Markdown", description="Current paper draft."),
-            self._file_entry("paper/writing_plan.md", label="Writing Plan", description="Paper writing plan."),
-            self._file_entry("paper/references.bib", label="References", description="Bibliography file."),
-            self._file_entry("paper/claim_evidence_map.json", label="Claim-Evidence Map", description="Claim to evidence mapping."),
-            self._file_entry("paper/baseline_inventory.json", label="Baseline Inventory", description="Canonical and supplementary baseline inventory for writing."),
-            self._file_entry("paper/build/compile_report.json", label="Compile Report", description="Paper build/compile report."),
-            self._file_entry("paper/paper_bundle_manifest.json", label="Bundle Manifest", description="Final paper bundle manifest."),
-            self._file_entry("release/open_source/manifest.json", label="Open Source Manifest", description="Open-source cleanup and release preparation manifest."),
-            self._file_entry("release/open_source/cleanup_plan.md", label="Open Source Cleanup Plan", description="Checklist for cleaning the paper branch into a public release."),
+            self._file_entry(paper_root / "selected_outline.json", label="Selected Outline", description="Chosen paper outline."),
+            self._file_entry(paper_root / "outline_selection.md", label="Outline Selection Note", description="Outline selection rationale."),
+            self._file_entry(paper_root / "draft.md", label="Draft Markdown", description="Current paper draft."),
+            self._file_entry(paper_root / "writing_plan.md", label="Writing Plan", description="Paper writing plan."),
+            self._file_entry(paper_root / "references.bib", label="References", description="Bibliography file."),
+            self._file_entry(paper_root / "claim_evidence_map.json", label="Claim-Evidence Map", description="Claim to evidence mapping."),
+            self._file_entry(paper_root / "baseline_inventory.json", label="Baseline Inventory", description="Canonical and supplementary baseline inventory for writing."),
+            self._file_entry(paper_root / "build" / "compile_report.json", label="Compile Report", description="Paper build/compile report."),
+            self._file_entry(paper_root / "paper_bundle_manifest.json", label="Bundle Manifest", description="Final paper bundle manifest."),
+            self._file_entry(open_source_root / "manifest.json", label="Open Source Manifest", description="Open-source cleanup and release preparation manifest."),
+            self._file_entry(open_source_root / "cleanup_plan.md", label="Open Source Cleanup Plan", description="Checklist for cleaning the paper branch into a public release."),
             self._file_entry(latex_root_rel, label="LaTeX Sources", description="LaTeX source folder.", expected_kind="directory"),
             self._file_entry(main_tex_rel, label="Main TeX", description="Primary TeX source file."),
         ]
@@ -1290,7 +1331,7 @@ class QuestStageViewBuilder:
 
     def _paper_candidates(self) -> list[dict[str, Any]]:
         candidates: list[dict[str, Any]] = []
-        for path in sorted((self.quest_root / "paper" / "outlines" / "candidates").glob("*.json")):
+        for path in sorted((self._paper_root() / "outlines" / "candidates").glob("*.json")):
             payload = read_json(path, {})
             if not isinstance(payload, dict):
                 payload = {}
@@ -1319,7 +1360,8 @@ class QuestStageViewBuilder:
             )
             and self._branch_matches(self._payload(item), allow_parent=True)
         ]
-        selected_outline_path = self.quest_root / "paper" / "selected_outline.json"
+        paper_root = self._paper_root()
+        selected_outline_path = paper_root / "selected_outline.json"
         selected_outline = read_json(selected_outline_path, {}) if selected_outline_path.exists() else {}
         if not isinstance(selected_outline, dict):
             selected_outline = {}
@@ -1329,15 +1371,18 @@ class QuestStageViewBuilder:
             else {}
         )
         candidates = self._paper_candidates()
-        compile_report = read_json(self.quest_root / "paper" / "build" / "compile_report.json", {})
+        compile_report = read_json(paper_root / "build" / "compile_report.json", {})
         if not isinstance(compile_report, dict):
             compile_report = {}
         bundle_manifest = self._paper_bundle_manifest()
         latex_root_rel = self._paper_latex_root(bundle_manifest)
         main_tex_rel = self._paper_main_tex(latex_root_rel)
-        references_bib = read_text(self.quest_root / "paper" / "references.bib", "")
+        references_bib = read_text(paper_root / "references.bib", "")
         references_count = sum(1 for line in references_bib.splitlines() if line.lstrip().startswith("@"))
         pdf_paths = self._paper_pdf_candidates(bundle_manifest, main_tex_rel=main_tex_rel)
+        draft_rel = self._paper_relative_path(paper_root / "draft.md") or "paper/draft.md"
+        writing_plan_rel = self._paper_relative_path(paper_root / "writing_plan.md") or "paper/writing_plan.md"
+        claim_map_rel = self._paper_relative_path(paper_root / "claim_evidence_map.json") or "paper/claim_evidence_map.json"
         selected_title = str(
             detailed.get("title") or selected_outline.get("title") or bundle_manifest.get("title") or "Drafting"
         ).strip() or "Drafting"
@@ -1356,7 +1401,7 @@ class QuestStageViewBuilder:
             overview=[
                 _field("Selected Outline", selected_title if selected_outline else "Not selected"),
                 _field("Candidate Count", len(candidates)),
-                _field("Draft Status", "present" if (self.quest_root / "paper" / "draft.md").exists() else "missing"),
+                _field("Draft Status", "present" if (paper_root / "draft.md").exists() else "missing"),
                 _field("Bundle Status", "present" if bundle_manifest else "missing"),
             ],
             key_facts=[
@@ -1377,10 +1422,10 @@ class QuestStageViewBuilder:
                     "outline_candidates": candidates,
                     "selected_outline": selected_outline,
                     "drafting": {
-                        "writing_plan_path": "paper/writing_plan.md",
-                        "draft_path": "paper/draft.md",
+                        "writing_plan_path": writing_plan_rel,
+                        "draft_path": draft_rel,
                         "references_count": references_count,
-                        "claim_evidence_map_path": "paper/claim_evidence_map.json",
+                        "claim_evidence_map_path": claim_map_rel,
                     },
                     "build": {
                         "compile_report": compile_report,

package/src/deepscientist/runners/codex.py

@@ -771,7 +771,7 @@ class CodexRunner:
         ]
         if request.approval_policy:
             command.extend(["-c", f'approval_policy="{request.approval_policy}"'])
-        reasoning_effort = request.reasoning_effort if request.reasoning_effort is not None else "xhigh"
+        reasoning_effort = request.reasoning_effort
         if reasoning_effort:
             command.extend(["-c", f'model_reasoning_effort="{reasoning_effort}"'])
         tool_timeout_sec = self._positive_timeout_seconds(resolved_runner_config.get("mcp_tool_timeout_sec"))

package/src/prompts/connectors/qq.md

@@ -10,7 +10,7 @@
 - qq_summary_first_rule: start with the conclusion the user cares about, then what it means, then the next action
 - qq_progress_shape_rule: make the current task, the main difficulty or latest real progress, and the next concrete measure explicit whenever possible
 - qq_eta_rule: for baseline reproduction, main experiments, analysis experiments, and other important long-running research phases, include a rough ETA for the next meaningful result or the next update; if uncertain, say that and still give the next check-in window
-- qq_tool_call_keepalive_rule: for ordinary active work, if roughly 10 to 30 tool calls pass without a user-visible checkpoint, send one concise QQ progress update before continuing
+- qq_tool_call_keepalive_rule: for ordinary active work, prefer one concise QQ progress update after roughly 10 tool calls when there is already a human-meaningful delta, and do not let work drift beyond roughly 20 tool calls or about 15 minutes without a user-visible checkpoint
 - qq_internal_detail_rule: omit worker names, heartbeat timestamps, retry counters, pending/running/completed counts, file names, and monitor-window narration unless the user asked for them or the detail changes the recommended action
 - qq_translation_rule: convert internal execution and file-management work into user value, such as saying the baseline record is now organized for easier later comparison instead of listing touched files
 - qq_preflight_rule: before sending a QQ progress update, rewrite it if it still sounds like a monitoring log, execution diary, or file inventory

package/src/prompts/contracts/shared_interaction.md

@@ -0,0 +1,14 @@
+# Shared Interaction Contract
+
+This shared contract is injected once per turn and applies across the stage and companion skills that use `artifact.interact(...)` as the main user-visible continuity channel.
+
+## Shared interaction rules
+
+- Treat `artifact.interact(...)` as the main long-lived communication thread across TUI, web, and bound connectors.
+- If `artifact.interact(...)` returns queued user requirements, treat them as the highest-priority user instruction bundle before continuing the current stage or companion-skill task.
+- Immediately follow any non-empty mailbox poll with another `artifact.interact(...)` update that confirms receipt; if the request is directly answerable, answer there, otherwise say the current subtask is paused, give a short plan plus nearest report-back point, and handle that request first.
+- Emit `artifact.interact(kind='progress', reply_mode='threaded', ...)` when there is real user-visible progress: a meaningful checkpoint, route-shaping update, or a concise keepalive once active work has crossed roughly 10 tool calls with a human-meaningful delta. Do not let ordinary active work drift beyond roughly 20 tool calls or about 15 minutes without a user-visible update.
+- Keep progress updates chat-like and easy to understand: say what changed, what it means, and what happens next.
+- Default to plain-language summaries. Do not mention file paths, artifact ids, branch/worktree ids, session ids, raw commands, or raw logs unless the user asks or needs them to act.
+- Use `reply_mode='blocking'` only for real user decisions that cannot be resolved from local evidence.
+- For any blocking decision request, provide 1 to 3 concrete options, put the recommended option first, explain each option's actual content plus pros and cons, and wait up to 1 day when feasible. If the blocker is a missing external credential or secret that only the user can provide, keep the quest waiting, ask the user to supply it or choose an alternative, and do not self-resolve; if resumed without that credential and no other work is possible, a long low-frequency wait such as `bash_exec(command='sleep 3600', mode='await', timeout_seconds=3700)` is acceptable. Otherwise choose the best option yourself and notify the user of the chosen option if the timeout expires.