@researai/deepscientist 1.5.7 → 1.5.9

package/src/deepscientist/quest/node_traces.py

@@ -46,8 +46,10 @@ def _infer_stage_from_skill(skill_id: object) -> str | None:
         return "baseline"
     if normalized in {"idea", "scout+idea"}:
         return "idea"
-    if normalized in {"experiment", "analysis-campaign", "analysis"}:
+    if normalized == "experiment":
         return "experiment"
+    if normalized in {"analysis-campaign", "analysis", "analysis_slice"}:
+        return "analysis"
     if normalized in {"write", "finalize"}:
         return "writing"
     if normalized == "decision":
@@ -103,6 +105,56 @@ def _load_artifact_record(path: Path) -> dict[str, Any] | None:
     return None
 
 
+def _normalize_paths_map(value: object) -> dict[str, str | None]:
+    if not isinstance(value, dict):
+        return {}
+    normalized: dict[str, str | None] = {}
+    for raw_key, raw_value in value.items():
+        key = str(raw_key or "").strip()
+        if not key:
+            continue
+        if raw_value is None:
+            normalized[key] = None
+            continue
+        text = str(raw_value).strip()
+        normalized[key] = text or None
+    return normalized
+
+
+def _normalize_string_list(value: object) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    items: list[str] = []
+    seen: set[str] = set()
+    for raw in value:
+        text = str(raw or "").strip()
+        if not text or text in seen:
+            continue
+        seen.add(text)
+        items.append(text)
+    return items
+
+
+def _run_artifact_context(quest_root: Path, run_id: str | None) -> dict[str, Any] | None:
+    normalized = str(run_id or "").strip()
+    if not normalized:
+        return None
+    path = quest_root / ".ds" / "runs" / normalized / "artifact.json"
+    payload = read_json(path, {})
+    if not isinstance(payload, dict) or not payload:
+        return None
+    record = dict(payload.get("record") or {}) if isinstance(payload.get("record"), dict) else {}
+    checkpoint = dict(payload.get("checkpoint") or {}) if isinstance(payload.get("checkpoint"), dict) else {}
+    return {
+        "path": str(path),
+        "record": record,
+        "checkpoint": checkpoint,
+        "head_commit": str(checkpoint.get("head") or record.get("head_commit") or "").strip() or None,
+        "paths_map": _normalize_paths_map(record.get("paths")),
+        "changed_files": _normalize_string_list(record.get("files_changed")),
+    }
+
+
 def _build_run_contexts(quest_root: Path, *, default_branch: str) -> dict[str, dict[str, Any]]:
     contexts: dict[str, dict[str, Any]] = {}
     artifact_roots = [quest_root / "artifacts"]
@@ -145,6 +197,7 @@ def _build_run_contexts(quest_root: Path, *, default_branch: str) -> dict[str, d
             current["worktree_rel_path"] = current.get("worktree_rel_path") or record.get("worktree_rel_path")
             current["summary"] = current.get("summary") or summary
             current["updated_at"] = current.get("updated_at") or updated_at
+            current["artifact_path"] = current.get("artifact_path") or str(artifact_path)
             contexts[run_id] = current
     return contexts
 
@@ -160,6 +213,7 @@ def _resolve_entry_context(
     raw_event_type = str(entry.get("raw_event_type") or entry.get("kind") or "").strip()
     run_context = run_contexts.get(run_id or "")
     artifact_context: dict[str, Any] | None = None
+    artifact_path: str | None = None
     for raw_path in entry.get("paths") or []:
         try:
             path = Path(str(raw_path))
@@ -169,14 +223,21 @@ def _resolve_entry_context(
             continue
         artifact_context = _load_artifact_record(path)
         if artifact_context:
+            artifact_path = str(path)
             break
+    if run_id is None and artifact_context:
+        run_id = str(artifact_context.get("run_id") or "").strip() or None
+    run_artifact = _run_artifact_context(quest_root, run_id)
     branch_name = _normalize_branch_name(
-        (artifact_context or {}).get("branch") or (run_context or {}).get("branch_name"),
+        (artifact_context or {}).get("branch")
+        or (((run_artifact or {}).get("record") or {}) if isinstance((run_artifact or {}).get("record"), dict) else {}).get("branch")
+        or (run_context or {}).get("branch_name"),
         fallback=default_branch,
     )
     stage_key = (
         _infer_stage_from_skill(entry.get("skill_id"))
         or _infer_stage_from_artifact(artifact_context or {})
+        or _infer_stage_from_artifact((((run_artifact or {}).get("record") or {}) if run_artifact else {}))
         or (run_context or {}).get("stage_key")
         or _infer_stage_from_event_type(raw_event_type)
         or "general"
@@ -186,9 +247,15 @@ def _resolve_entry_context(
         "branch_name": branch_name,
         "stage_key": stage_key,
         "worktree_rel_path": (artifact_context or {}).get("worktree_rel_path")
+        or (((run_artifact or {}).get("record") or {}) if run_artifact else {}).get("worktree_rel_path")
         or (run_context or {}).get("worktree_rel_path"),
+        "artifact_context": artifact_context,
+        "artifact_path": artifact_path or (run_context or {}).get("artifact_path"),
+        "run_artifact": run_artifact,
         "trace_confidence": "artifact"
         if artifact_context
+        else "run_artifact"
+        if run_artifact
         else "run_context"
         if run_context
         else "default_branch",
@@ -196,6 +263,25 @@ def _resolve_entry_context(
 
 
 def _build_action(entry: dict[str, Any], context: dict[str, Any]) -> dict[str, Any]:
+    artifact_context = (
+        dict(context.get("artifact_context") or {})
+        if isinstance(context.get("artifact_context"), dict)
+        else {}
+    )
+    run_artifact = (
+        dict(context.get("run_artifact") or {})
+        if isinstance(context.get("run_artifact"), dict)
+        else {}
+    )
+    run_record = dict(run_artifact.get("record") or {}) if isinstance(run_artifact.get("record"), dict) else {}
+    paths_map = _normalize_paths_map(artifact_context.get("paths")) or _normalize_paths_map(run_record.get("paths"))
+    changed_files = _normalize_string_list(artifact_context.get("files_changed")) or _normalize_string_list(
+        artifact_context.get("changed_files")
+    ) or _normalize_string_list(run_record.get("files_changed")) or _normalize_string_list(run_artifact.get("changed_files"))
+    details_json = dict(artifact_context.get("details") or {}) if isinstance(artifact_context.get("details"), dict) else {}
+    checkpoint_json = dict(run_artifact.get("checkpoint") or {}) if isinstance(run_artifact.get("checkpoint"), dict) else {}
+    metadata = dict(entry.get("metadata") or {}) if isinstance(entry.get("metadata"), dict) else {}
+    payload_json = artifact_context or metadata or None
     return {
         "action_id": entry.get("id"),
         "kind": entry.get("kind"),
@@ -217,6 +303,22 @@ def _build_action(entry: dict[str, Any], context: dict[str, Any]) -> dict[str, A
         "reason": entry.get("reason"),
         "raw_event_type": entry.get("raw_event_type"),
         "paths": [str(item) for item in (entry.get("paths") or []) if item],
+        "paths_map": paths_map or None,
+        "artifact_id": artifact_context.get("artifact_id"),
+        "artifact_kind": artifact_context.get("kind"),
+        "artifact_path": context.get("artifact_path"),
+        "head_commit": (
+            str(
+                artifact_context.get("head_commit")
+                or run_artifact.get("head_commit")
+                or ""
+            ).strip()
+            or None
+        ),
+        "payload_json": payload_json,
+        "details_json": details_json or (metadata or None),
+        "checkpoint_json": checkpoint_json or None,
+        "changed_files": changed_files or None,
         "trace_confidence": context.get("trace_confidence"),
     }
 
@@ -277,6 +379,24 @@ def _build_trace_item(
     ) or None
     run_ids = sorted({str(item.get("run_id") or "").strip() for item in ordered_actions if item.get("run_id")})
     skill_ids = sorted({str(item.get("skill_id") or "").strip() for item in ordered_actions if item.get("skill_id")})
+    primary_action = next(
+        (
+            action
+            for action in reversed(ordered_actions)
+            if action.get("artifact_id")
+            or action.get("head_commit")
+            or action.get("payload_json")
+            or action.get("details_json")
+        ),
+        ordered_actions[-1] if ordered_actions else {},
+    )
+    merged_paths_map: dict[str, str | None] = {}
+    for action in ordered_actions:
+        if isinstance(action.get("paths_map"), dict):
+            merged_paths_map.update(action.get("paths_map") or {})
+    merged_changed_files = _normalize_string_list(
+        [item for action in ordered_actions for item in (action.get("changed_files") or [])]
+    )
     return {
         "selection_type": selection_type,
         "selection_ref": selection_ref,
@@ -291,6 +411,13 @@ def _build_trace_item(
         "counts": _build_counts(ordered_actions),
         "run_ids": run_ids,
         "skill_ids": skill_ids,
+        "artifact_id": primary_action.get("artifact_id"),
+        "artifact_kind": primary_action.get("artifact_kind"),
+        "head_commit": primary_action.get("head_commit"),
+        "payload_json": primary_action.get("payload_json"),
+        "details_json": primary_action.get("details_json"),
+        "paths_map": merged_paths_map or None,
+        "changed_files": merged_changed_files or None,
         "actions": ordered_actions,
     }
 

package/src/deepscientist/quest/service.py

@@ -17,11 +17,12 @@ try:
 except ImportError:  # pragma: no cover
     fcntl = None
 
-from ..artifact.metrics import build_metrics_timeline, normalize_metrics_summary
+from ..artifact.metrics import build_metrics_timeline, extract_latest_metric
 from ..config import ConfigManager
 from ..connector_runtime import conversation_identity_key, normalize_conversation_id
 from ..gitops import current_branch, export_git_graph, head_commit, init_repo
 from ..home import repo_root
+from ..registries import BaselineRegistry
 from ..shared import append_jsonl, ensure_dir, generate_id, read_json, read_jsonl, read_text, read_yaml, resolve_within, run_command, sha256_text, slugify, utc_now, write_json, write_text, write_yaml
 from ..skills import SkillInstaller
 from ..web_search import extract_web_search_payload
@@ -48,6 +49,7 @@ class QuestService:
         self.home = home
         self.quests_root = home / "quests"
         self.skill_installer = skill_installer
+        self.baseline_registry = BaselineRegistry(home)
         self._file_cache_lock = threading.Lock()
         self._file_cache: dict[str, dict[str, Any]] = {}
         self._jsonl_cache_lock = threading.Lock()
@@ -116,6 +118,10 @@ class QuestService:
     def _research_state_path(quest_root: Path) -> Path:
         return quest_root / ".ds" / "research_state.json"
 
+    @staticmethod
+    def _lab_canvas_state_path(quest_root: Path) -> Path:
+        return quest_root / ".ds" / "lab_canvas_state.json"
+
     def _default_research_state(self, quest_root: Path) -> dict[str, Any]:
         return {
             "version": 1,
@@ -129,12 +135,27 @@ class QuestService:
             "active_analysis_campaign_id": None,
             "analysis_parent_branch": None,
             "analysis_parent_worktree_root": None,
+            "paper_parent_branch": None,
+            "paper_parent_worktree_root": None,
+            "paper_parent_run_id": None,
             "next_pending_slice_id": None,
             "workspace_mode": "quest",
             "last_flow_type": None,
             "updated_at": utc_now(),
         }
 
+    def _default_lab_canvas_state(self, quest_root: Path) -> dict[str, Any]:
+        return {
+            "version": 1,
+            "layout_json": {
+                "branch": {},
+                "event": {},
+                "stage": {},
+                "preferences": {},
+            },
+            "updated_at": utc_now(),
+        }
+
     def read_research_state(self, quest_root: Path) -> dict[str, Any]:
         self._initialize_runtime_files(quest_root)
         defaults = self._default_research_state(quest_root)
@@ -151,6 +172,9 @@ class QuestService:
         parent_root = str(merged.get("analysis_parent_worktree_root") or "").strip()
         if parent_root and not Path(parent_root).exists():
             merged["analysis_parent_worktree_root"] = None
+        paper_parent_root = str(merged.get("paper_parent_worktree_root") or "").strip()
+        if paper_parent_root and not Path(paper_parent_root).exists():
+            merged["paper_parent_worktree_root"] = None
         return merged
 
     def write_research_state(self, quest_root: Path, payload: dict[str, Any]) -> dict[str, Any]:
@@ -166,6 +190,39 @@ class QuestService:
             current[key] = str(value) if isinstance(value, Path) else value
         return self.write_research_state(quest_root, current)
 
+    def read_lab_canvas_state(self, quest_root: Path) -> dict[str, Any]:
+        self._initialize_runtime_files(quest_root)
+        defaults = self._default_lab_canvas_state(quest_root)
+        payload = self._read_cached_json(self._lab_canvas_state_path(quest_root), defaults)
+        if not isinstance(payload, dict):
+            payload = defaults
+        merged = {**defaults, **payload}
+        layout_json = dict(merged.get("layout_json") or {}) if isinstance(merged.get("layout_json"), dict) else {}
+        for key in ("branch", "event", "stage", "preferences"):
+            if not isinstance(layout_json.get(key), dict):
+                layout_json[key] = {}
+        merged["layout_json"] = layout_json
+        return merged
+
+    def update_lab_canvas_state(
+        self,
+        quest_root: Path,
+        *,
+        layout_json: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        current = self.read_lab_canvas_state(quest_root)
+        normalized_layout = dict(layout_json or {}) if isinstance(layout_json, dict) else {}
+        for key in ("branch", "event", "stage", "preferences"):
+            if not isinstance(normalized_layout.get(key), dict):
+                normalized_layout[key] = {}
+        payload = {
+            **current,
+            "layout_json": normalized_layout,
+            "updated_at": utc_now(),
+        }
+        write_json(self._lab_canvas_state_path(quest_root), payload)
+        return payload
+
     def workspace_roots(self, quest_root: Path) -> list[Path]:
         roots: list[Path] = [quest_root]
         state = self.read_research_state(quest_root)
@@ -206,9 +263,37 @@ class QuestService:
     def _artifact_roots(self, quest_root: Path) -> list[Path]:
         return [root for root in self.workspace_roots(quest_root) if (root / "artifacts").exists()]
 
+    @staticmethod
+    def _artifact_item_identity(path: Path, payload: dict[str, Any], *, kind: str) -> str:
+        normalized_kind = str(kind or payload.get("kind") or path.parent.name or "artifact").strip() or "artifact"
+        artifact_id = str(payload.get("artifact_id") or payload.get("id") or "").strip()
+        if artifact_id:
+            return f"{normalized_kind}:artifact:{artifact_id}"
+        branch_name = str(payload.get("branch") or "").strip()
+        run_id = str(payload.get("run_id") or "").strip()
+        if normalized_kind == "runs" and run_id and branch_name:
+            return f"{normalized_kind}:branch_run:{branch_name}:{run_id}"
+        if normalized_kind == "runs" and run_id:
+            return f"{normalized_kind}:run:{run_id}"
+        idea_id = str(payload.get("idea_id") or "").strip()
+        if normalized_kind == "ideas" and idea_id and branch_name:
+            return f"{normalized_kind}:branch_idea:{branch_name}:{idea_id}"
+        if normalized_kind == "ideas" and idea_id:
+            return f"{normalized_kind}:idea:{idea_id}"
+        return f"path:{path.resolve()}"
+
+    @staticmethod
+    def _artifact_item_rank(payload: dict[str, Any], *, path: Path, mtime_ns: int) -> tuple[str, str, int, int, str]:
+        return (
+            str(payload.get("updated_at") or ""),
+            str(payload.get("created_at") or ""),
+            len(payload),
+            mtime_ns,
+            str(path),
+        )
+
     def _collect_artifacts(self, quest_root: Path) -> list[dict[str, Any]]:
-        artifacts: list[dict[str, Any]] = []
-        seen_paths: set[str] = set()
+        artifacts_by_identity: dict[str, dict[str, Any]] = {}
         for root in self._artifact_roots(quest_root):
             artifacts_root = root / "artifacts"
             if not artifacts_root.exists():
@@ -217,19 +302,37 @@ class QuestService:
                 if not folder.is_dir():
                     continue
                 for path in sorted(folder.glob("*.json")):
-                    resolved_key = str(path.resolve())
-                    if resolved_key in seen_paths:
-                        continue
-                    seen_paths.add(resolved_key)
                     item = self._read_cached_json(path, {})
-                    artifacts.append(
-                        {
-                            "kind": folder.name,
-                            "path": str(path),
-                            "payload": item,
-                            "workspace_root": str(root),
-                        }
-                    )
+                    payload = item if isinstance(item, dict) else {}
+                    try:
+                        mtime_ns = path.stat().st_mtime_ns
+                    except OSError:
+                        mtime_ns = 0
+                    artifact = {
+                        "kind": folder.name,
+                        "path": str(path),
+                        "payload": item,
+                        "workspace_root": str(root),
+                    }
+                    identity = self._artifact_item_identity(path, payload, kind=folder.name)
+                    existing = artifacts_by_identity.get(identity)
+                    existing_payload = existing.get("payload") if isinstance((existing or {}).get("payload"), dict) else {}
+                    existing_path = Path(str((existing or {}).get("path") or path))
+                    try:
+                        existing_mtime_ns = existing_path.stat().st_mtime_ns if existing else 0
+                    except OSError:
+                        existing_mtime_ns = 0
+                    if existing is None or self._artifact_item_rank(
+                        payload,
+                        path=path,
+                        mtime_ns=mtime_ns,
+                    ) >= self._artifact_item_rank(
+                        existing_payload,
+                        path=existing_path,
+                        mtime_ns=existing_mtime_ns,
+                    ):
+                        artifacts_by_identity[identity] = artifact
+        artifacts = list(artifacts_by_identity.values())
         artifacts.sort(
             key=lambda item: str(
                 ((item.get("payload") or {}).get("updated_at"))
@@ -253,6 +356,9 @@ class QuestService:
                     continue
                 seen_paths.add(key)
                 payload = self._read_cached_yaml(path, {})
+                baseline_id = str(payload.get("source_baseline_id") or "").strip() if isinstance(payload, dict) else ""
+                if baseline_id and self.baseline_registry.is_deleted(baseline_id):
+                    continue
                 if isinstance(payload, dict) and payload:
                     attachments.append(payload)
         if not attachments:
@@ -267,22 +373,7 @@ class QuestService:
 
     @staticmethod
     def _latest_metric_from_payload(payload: dict[str, Any]) -> dict[str, Any] | None:
-        metrics_summary = normalize_metrics_summary(payload.get("metrics_summary"))
-        if not metrics_summary:
-            return None
-        progress_eval = payload.get("progress_eval") if isinstance(payload.get("progress_eval"), dict) else {}
-        comparisons = payload.get("baseline_comparisons") if isinstance(payload.get("baseline_comparisons"), dict) else {}
-        primary_metric_id = (
-            str(progress_eval.get("primary_metric_id") or comparisons.get("primary_metric_id") or "").strip()
-            or next(iter(metrics_summary.keys()))
-        )
-        result = {
-            "key": primary_metric_id,
-            "value": metrics_summary.get(primary_metric_id),
-        }
-        if progress_eval.get("delta_vs_baseline") is not None:
-            result["delta_vs_baseline"] = progress_eval.get("delta_vs_baseline")
-        return result
+        return extract_latest_metric(payload)
 
     @staticmethod
     def _parse_numeric_quest_id(value: str | None) -> int | None:
@@ -980,6 +1071,9 @@ class QuestService:
             "active_analysis_campaign_id": research_state.get("active_analysis_campaign_id"),
             "analysis_parent_branch": research_state.get("analysis_parent_branch"),
             "analysis_parent_worktree_root": research_state.get("analysis_parent_worktree_root"),
+            "paper_parent_branch": research_state.get("paper_parent_branch"),
+            "paper_parent_worktree_root": research_state.get("paper_parent_worktree_root"),
+            "paper_parent_run_id": research_state.get("paper_parent_run_id"),
             "next_pending_slice_id": research_state.get("next_pending_slice_id"),
             "workspace_mode": research_state.get("workspace_mode") or "quest",
             "active_baseline_id": active_baseline_id,
@@ -1981,7 +2075,11 @@ class QuestService:
                 },
             }
 
-        resolution_root = quest_root if document_id.startswith("questpath::") else workspace_root
+        resolution_root = (
+            quest_root
+            if document_id.startswith(("questpath::", "memory::"))
+            else workspace_root
+        )
         path, writable, scope, source_kind = self._resolve_document(resolution_root, document_id)
         renderer_hint, mime_type = self._renderer_hint_for(path)
         is_text = self._is_text_document(path, mime_type, renderer_hint)
@@ -2162,11 +2260,16 @@ class QuestService:
         asset_name = f"{safe_stem}-{generate_id('asset').split('-', 1)[1]}{asset_suffix}"
         asset_relative_dir = self._markdown_asset_directory(base_relative)
         asset_relative = (asset_relative_dir / asset_name).as_posix()
-        asset_root = quest_root if document_id.startswith("questpath::") else workspace_root
+        asset_root = (
+            quest_root
+            if document_id.startswith(("questpath::", "memory::"))
+            else workspace_root
+        )
         asset_path = resolve_within(asset_root, asset_relative)
         ensure_dir(asset_path.parent)
         asset_path.write_bytes(content)
-        asset_document_id = f"{'questpath' if document_id.startswith('questpath::') else 'path'}::{asset_relative}"
+        asset_document_scope = "questpath" if document_id.startswith(("questpath::", "memory::")) else "path"
+        asset_document_id = f"{asset_document_scope}::{asset_relative}"
         relative_markdown_path = self._relative_path_from_base(base_relative, asset_relative)
         return {
             "ok": True,
@@ -2368,6 +2471,9 @@ class QuestService:
         research_state_path = self._research_state_path(quest_root)
         if not research_state_path.exists():
             write_json(research_state_path, self._default_research_state(quest_root))
+        lab_canvas_state_path = self._lab_canvas_state_path(quest_root)
+        if not lab_canvas_state_path.exists():
+            write_json(lab_canvas_state_path, self._default_lab_canvas_state(quest_root))
         agent_status_path = self._agent_status_path(quest_root)
         if not agent_status_path.exists():
             write_json(agent_status_path, self._default_agent_status(quest_root))