@researai/deepscientist 1.5.14 → 1.5.16

package/src/deepscientist/doctor.py

@@ -11,6 +11,7 @@ from typing import Any
 from urllib.error import URLError
 from urllib.request import Request, urlopen
 
+from .bash_exec.shells import build_exec_shell_launch, build_terminal_shell_launch
 from .config import ConfigManager
 from .home import ensure_home_layout
 from .runtime_tools import RuntimeToolService
@@ -321,6 +322,55 @@ def _check_bundles(repo_root: Path) -> dict[str, Any]:
     )
 
 
+def _check_shell_backend() -> dict[str, Any]:
+    exec_launch = build_exec_shell_launch("echo ok")
+    terminal_launch = build_terminal_shell_launch(Path("doctor-terminal-probe"))
+
+    def resolve_binary(binary: str) -> str | None:
+        candidate = str(binary or "").strip()
+        if not candidate:
+            return None
+        if os.path.isabs(candidate) or os.path.sep in candidate or (os.path.altsep and os.path.altsep in candidate):
+            return candidate if Path(candidate).exists() else None
+        return which(candidate)
+
+    details = {
+        "exec_shell": exec_launch.shell_name,
+        "exec_shell_family": exec_launch.family,
+        "exec_argv": exec_launch.argv,
+        "terminal_shell": terminal_launch.shell_name,
+        "terminal_shell_family": terminal_launch.family,
+        "terminal_argv": terminal_launch.argv,
+    }
+    warnings: list[str] = []
+    guidance: list[str] = []
+    errors: list[str] = []
+
+    exec_binary = resolve_binary(exec_launch.argv[0])
+    terminal_binary = resolve_binary(terminal_launch.argv[0])
+    details["exec_resolved_binary"] = exec_binary
+    details["terminal_resolved_binary"] = terminal_binary
+
+    if sys.platform == "win32":
+        warnings.append("Native Windows support is currently experimental; WSL2 remains the most battle-tested path.")
+        if not exec_binary:
+            errors.append("DeepScientist could not resolve a Windows command shell for bash_exec.")
+            guidance.append("Install PowerShell (`pwsh`) or ensure `powershell.exe` is available on PATH.")
+        if not terminal_binary:
+            errors.append("DeepScientist could not resolve a Windows interactive shell backend.")
+            guidance.append("Ensure `powershell.exe` is available on PATH for the interactive terminal surface.")
+    return _make_check(
+        check_id="shell_backend",
+        label="Shell backend",
+        ok=len(errors) == 0,
+        summary="DeepScientist resolved platform shell backends for command and terminal sessions." if len(errors) == 0 else "DeepScientist could not resolve a required shell backend.",
+        warnings=warnings,
+        errors=errors,
+        guidance=guidance,
+        details=details,
+    )
+
+
 def _check_latex_runtime(home: Path) -> dict[str, Any]:
     runtime = RuntimeToolService(home).status("tinytex")
     pdflatex = runtime.get("binaries", {}).get("pdflatex") or {}
@@ -436,6 +486,7 @@ def run_doctor(home: Path, *, repo_root: Path) -> dict[str, Any]:
         _check_python_runtime(),
         _check_home_writable(home),
         _check_uv(home),
+        _check_shell_backend(),
         _check_git(config_manager),
         _check_config_validation(config_manager),
         _check_runner_support(config_manager),

package/src/deepscientist/file_lock.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+import os
+from pathlib import Path
+from typing import Iterator, TextIO
+
+if os.name == "nt":  # pragma: no cover - exercised on Windows
+    import msvcrt
+else:  # pragma: no cover - exercised on POSIX
+    import fcntl
+
+
+def _ensure_lockable_file(handle: TextIO) -> None:
+    handle.seek(0, os.SEEK_END)
+    if handle.tell() > 0:
+        handle.seek(0)
+        return
+    handle.write("\0")
+    handle.flush()
+    handle.seek(0)
+
+
+def _lock_handle(handle: TextIO) -> None:
+    if os.name == "nt":  # pragma: no cover - exercised on Windows
+        _ensure_lockable_file(handle)
+        msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
+        return
+    fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
+
+
+def _unlock_handle(handle: TextIO) -> None:
+    if os.name == "nt":  # pragma: no cover - exercised on Windows
+        handle.seek(0)
+        msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
+        return
+    fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
+
+
+@contextmanager
+def advisory_file_lock(path: Path) -> Iterator[TextIO]:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a+", encoding="utf-8") as handle:
+        _lock_handle(handle)
+        try:
+            yield handle
+        finally:
+            _unlock_handle(handle)

package/src/deepscientist/gitops/__init__.py

@@ -1,4 +1,12 @@
-from .diff import commit_detail, compare_refs, diff_file_between_refs, diff_file_for_commit, list_branch_canvas, log_ref_history
+from .diff import (
+    commit_detail,
+    compare_refs,
+    diff_file_between_refs,
+    diff_file_for_commit,
+    list_branch_canvas,
+    list_commit_canvas,
+    log_ref_history,
+)
 from .graph import export_git_graph
 from .service import (
     branch_exists,
@@ -26,5 +34,6 @@ __all__ = [
     "head_commit",
     "init_repo",
     "list_branch_canvas",
+    "list_commit_canvas",
     "log_ref_history",
 ]

package/src/deepscientist/gitops/diff.py

@@ -5,7 +5,7 @@ from pathlib import Path
 from typing import Any
 
 from ..artifact.metrics import extract_latest_metric
-from ..shared import read_json, run_command
+from ..shared import read_json, run_command, slugify
 from .service import branch_exists, current_branch, head_commit
 
 
@@ -68,6 +68,12 @@ def list_branch_canvas(repo: Path, *, quest_id: str) -> dict[str, Any]:
             "run_id": state.get("run_id"),
             "run_kind": state.get("run_kind"),
             "idea_id": state.get("idea_id"),
+            "paper_line_id": state.get("paper_line_id"),
+            "paper_line_branch": state.get("paper_line_branch"),
+            "selected_outline_ref": state.get("selected_outline_ref"),
+            "source_branch": state.get("source_branch"),
+            "source_run_id": state.get("source_run_id"),
+            "source_idea_id": state.get("source_idea_id"),
             "parent_branch_recorded": state.get("parent_branch"),
             "worktree_root": state.get("worktree_root"),
             "latest_metric": state.get("latest_metric"),
@@ -107,6 +113,67 @@ def list_branch_canvas(repo: Path, *, quest_id: str) -> dict[str, Any]:
     }
 
 
+def list_commit_canvas(repo: Path, *, quest_id: str, limit: int = 80) -> dict[str, Any]:
+    resolved_limit = max(1, min(int(limit), 200))
+    head = head_commit(repo)
+    current_ref = current_branch(repo)
+    commits = _git_commit_canvas_log(repo, limit=resolved_limit)
+    nodes: list[dict[str, Any]] = []
+    edges: list[dict[str, Any]] = []
+    seen_shas = {item["sha"] for item in commits if str(item.get("sha") or "").strip()}
+
+    for commit in commits:
+        sha = str(commit.get("sha") or "").strip()
+        if not sha:
+            continue
+        detail = commit_detail(repo, sha=sha)
+        parents = [str(item).strip() for item in (detail.get("parents") or []) if str(item).strip()]
+        files = detail.get("files") or []
+        changed_paths = [
+            str(item.get("path") or "").strip()
+            for item in files
+            if str(item.get("path") or "").strip()
+        ]
+        node = {
+            "sha": sha,
+            "short_sha": str(detail.get("short_sha") or commit.get("short_sha") or sha[:7]).strip(),
+            "parents": parents,
+            "subject": str(detail.get("subject") or commit.get("subject") or "").strip() or sha[:7],
+            "body_preview": _body_preview(str(detail.get("body") or "").strip()),
+            "authored_at": str(detail.get("authored_at") or commit.get("authored_at") or "").strip() or None,
+            "author_name": str(detail.get("author_name") or commit.get("author_name") or "").strip() or None,
+            "branch_refs": _normalize_branch_refs(commit.get("decorations")),
+            "current": bool(head and sha == head),
+            "active_workspace": bool(head and sha == head),
+            "changed_paths": changed_paths,
+            "file_count": int(detail.get("file_count") or len(files)),
+            "added": int(((detail.get("stats") or {}) if isinstance(detail.get("stats"), dict) else {}).get("added") or 0),
+            "removed": int(((detail.get("stats") or {}) if isinstance(detail.get("stats"), dict) else {}).get("removed") or 0),
+            "compare_base": parents[0] if parents else None,
+            "compare_head": sha,
+            "selection_type": "git_commit_node",
+        }
+        nodes.append(node)
+        for parent in parents:
+            if parent in seen_shas:
+                edges.append(
+                    {
+                        "from": parent,
+                        "to": sha,
+                        "relation": "parent",
+                    }
+                )
+
+    return {
+        "quest_id": quest_id,
+        "workspace_mode": "copilot",
+        "head": head,
+        "current_ref": current_ref,
+        "nodes": nodes,
+        "edges": edges,
+    }
+
+
 def compare_refs(repo: Path, *, base: str, head: str) -> dict[str, Any]:
     _require_ref(repo, base)
     _require_ref(repo, head)
@@ -356,9 +423,158 @@ def _collect_branch_state(repo: Path) -> dict[str, dict[str, Any]]:
         for item in state.get("recent_artifacts", []):
             if isinstance(item, dict):
                 item.pop("_sort_key", None)
+    for workspace_root in _canvas_workspace_roots(repo):
+        state_path = workspace_root / "paper" / "paper_line_state.json"
+        if not state_path.exists():
+            continue
+        payload = read_json(state_path, {})
+        if not isinstance(payload, dict) or not payload:
+            continue
+        paper_branch = str(payload.get("paper_branch") or "").strip() or current_branch(workspace_root)
+        if not paper_branch:
+            continue
+        state = branch_state[paper_branch]
+        state.setdefault("branch", paper_branch)
+        state["worktree_root"] = str(workspace_root)
+        state["paper_line_id"] = str(payload.get("paper_line_id") or "").strip() or state.get("paper_line_id")
+        state["paper_line_branch"] = paper_branch
+        state["selected_outline_ref"] = str(payload.get("selected_outline_ref") or "").strip() or state.get("selected_outline_ref")
+        state["source_branch"] = str(payload.get("source_branch") or "").strip() or state.get("source_branch")
+        state["source_run_id"] = str(payload.get("source_run_id") or "").strip() or state.get("source_run_id")
+        state["source_idea_id"] = str(payload.get("source_idea_id") or "").strip() or state.get("source_idea_id")
+        state["updated_at"] = str(payload.get("updated_at") or state.get("updated_at") or "")
+        if not state.get("parent_branch") and state.get("source_branch"):
+            state["parent_branch"] = state.get("source_branch")
+    for workspace_root in _canvas_workspace_roots(repo):
+        paper_root = workspace_root / "paper"
+        if not paper_root.exists():
+            continue
+        state_path = paper_root / "paper_line_state.json"
+        if state_path.exists():
+            continue
+        selected_outline = read_json(paper_root / "selected_outline.json", {})
+        bundle_manifest = read_json(paper_root / "paper_bundle_manifest.json", {})
+        selected_outline = selected_outline if isinstance(selected_outline, dict) else {}
+        bundle_manifest = bundle_manifest if isinstance(bundle_manifest, dict) else {}
+        if not selected_outline and not bundle_manifest:
+            continue
+        paper_branch = str(bundle_manifest.get("paper_branch") or "").strip() or current_branch(workspace_root)
+        if not paper_branch:
+            continue
+        selected_outline_ref = str(
+            selected_outline.get("outline_id") or bundle_manifest.get("selected_outline_ref") or ""
+        ).strip() or None
+        source_run_id = str(bundle_manifest.get("source_run_id") or "").strip() or None
+        state = branch_state[paper_branch]
+        state.setdefault("branch", paper_branch)
+        state["worktree_root"] = str(workspace_root)
+        state["paper_line_id"] = state.get("paper_line_id") or slugify(
+            "::".join([paper_branch or "paper", selected_outline_ref or "outline", source_run_id or "run"]),
+            "paper-line",
+        )
+        state["paper_line_branch"] = paper_branch
+        state["selected_outline_ref"] = selected_outline_ref or state.get("selected_outline_ref")
+        state["source_branch"] = str(bundle_manifest.get("source_branch") or "").strip() or state.get("source_branch")
+        state["source_run_id"] = source_run_id or state.get("source_run_id")
+        state["source_idea_id"] = str(bundle_manifest.get("source_idea_id") or "").strip() or state.get("source_idea_id")
+        if not state.get("parent_branch") and state.get("source_branch"):
+            state["parent_branch"] = state.get("source_branch")
+    campaigns_root = repo / ".ds" / "analysis_campaigns"
+    if campaigns_root.exists():
+        for path in sorted(campaigns_root.glob("*.json")):
+            manifest = read_json(path, {})
+            if not isinstance(manifest, dict) or not manifest:
+                continue
+            campaign_id = str(manifest.get("campaign_id") or path.stem).strip() or path.stem
+            paper_line_id = str(manifest.get("paper_line_id") or "").strip() or None
+            paper_line_branch = str(manifest.get("paper_line_branch") or "").strip() or None
+            analysis_parent_branch = str(manifest.get("parent_branch") or "").strip() or None
+            selected_outline_ref = str(manifest.get("selected_outline_ref") or "").strip() or None
+            source_idea_id = str(manifest.get("active_idea_id") or "").strip() or None
+            if not paper_line_branch:
+                paper_line_branch = _infer_paper_line_branch_for_campaign(
+                    manifest,
+                    branch_state=branch_state,
+                )
+            for item in manifest.get("slices") or []:
+                if not isinstance(item, dict):
+                    continue
+                branch_name = str(item.get("branch") or "").strip()
+                if not branch_name:
+                    continue
+                state = branch_state[branch_name]
+                state.setdefault("branch", branch_name)
+                state["campaign_id"] = campaign_id
+                state["paper_line_id"] = paper_line_id or state.get("paper_line_id")
+                state["paper_line_branch"] = paper_line_branch or state.get("paper_line_branch")
+                state["analysis_parent_branch"] = analysis_parent_branch or state.get("analysis_parent_branch")
+                state["selected_outline_ref"] = selected_outline_ref or state.get("selected_outline_ref")
+                state["source_idea_id"] = source_idea_id or state.get("source_idea_id")
+                if item.get("worktree_root"):
+                    state["worktree_root"] = item.get("worktree_root")
     return branch_state
 
 
+def _canvas_workspace_roots(repo: Path) -> list[Path]:
+    roots: list[Path] = [repo]
+    research_state = read_json(repo / ".ds" / "research_state.json", {})
+    preferred_raw = str((research_state or {}).get("research_head_worktree_root") or "").strip()
+    if preferred_raw:
+        preferred = Path(preferred_raw)
+        if preferred.exists():
+            roots.append(preferred)
+    worktrees_root = repo / ".ds" / "worktrees"
+    if worktrees_root.exists():
+        roots.extend(path for path in sorted(worktrees_root.iterdir()) if path.is_dir())
+    deduped: list[Path] = []
+    seen: set[str] = set()
+    for root in roots:
+        key = str(root.resolve())
+        if key in seen:
+            continue
+        seen.add(key)
+        deduped.append(root)
+    return deduped
+
+
+def _infer_paper_line_branch_for_campaign(
+    manifest: dict[str, Any],
+    *,
+    branch_state: dict[str, dict[str, Any]],
+) -> str | None:
+    selected_outline_ref = str(manifest.get("selected_outline_ref") or "").strip() or None
+    source_idea_id = str(manifest.get("active_idea_id") or "").strip() or None
+    source_run_id = str(manifest.get("parent_run_id") or "").strip() or None
+    source_branch = str(manifest.get("parent_branch") or "").strip() or None
+    ranked: list[tuple[int, str]] = []
+    for branch_name, state in branch_state.items():
+        if not branch_name.startswith("paper/"):
+            continue
+        score = 0
+        candidate_outline = str(state.get("selected_outline_ref") or "").strip() or None
+        candidate_idea = str(state.get("source_idea_id") or "").strip() or None
+        candidate_run = str(state.get("source_run_id") or "").strip() or None
+        candidate_branch = str(state.get("source_branch") or "").strip() or None
+        if selected_outline_ref:
+            if candidate_outline != selected_outline_ref:
+                continue
+            score += 2
+        if source_idea_id and candidate_idea == source_idea_id:
+            score += 4
+        if source_run_id and candidate_run == source_run_id:
+            score += 3
+        if source_branch and candidate_branch == source_branch:
+            score += 2
+        if score > 0:
+            ranked.append((score, branch_name))
+    if not ranked:
+        return None
+    ranked.sort(key=lambda item: (item[0], item[1]), reverse=True)
+    if len(ranked) > 1 and ranked[0][0] == ranked[1][0]:
+        return None
+    return ranked[0][1]
+
+
 def _resolve_run_result_payload(repo: Path, record: dict[str, Any]) -> dict[str, Any]:
     details = dict(record.get("details") or {}) if isinstance(record.get("details"), dict) else {}
     paths = dict(record.get("paths") or {}) if isinstance(record.get("paths"), dict) else {}
@@ -469,9 +685,17 @@ def _infer_parent_ref(
 ) -> str | None:
     if ref == default_ref:
         return None
+    if classifications[ref]["branch_kind"] == "analysis":
+        paper_line_branch = str(state.get("paper_line_branch") or "").strip()
+        if paper_line_branch and paper_line_branch in refs and paper_line_branch != ref:
+            return paper_line_branch
     parent_branch = str(state.get("parent_branch") or "").strip()
     if parent_branch and parent_branch in refs and parent_branch != ref:
         return parent_branch
+    if classifications[ref]["branch_kind"] == "paper":
+        source_branch = str(state.get("source_branch") or "").strip()
+        if source_branch and source_branch in refs and source_branch != ref:
+            return source_branch
     if ref.startswith("idea/"):
         return default_ref
     if state.get("idea_id"):
@@ -479,6 +703,9 @@ def _infer_parent_ref(
         if candidate in refs:
             return candidate
     if classifications[ref]["branch_kind"] == "analysis":
+        analysis_parent_branch = str(state.get("analysis_parent_branch") or "").strip()
+        if analysis_parent_branch and analysis_parent_branch in refs and analysis_parent_branch != ref:
+            return analysis_parent_branch
         major_refs = [
             candidate
             for candidate, meta in classifications.items()
@@ -609,6 +836,74 @@ def _git_log(repo: Path, *, revspec: str, limit: int = 30) -> list[dict[str, Any
     return commits
 
 
+def _git_commit_canvas_log(repo: Path, *, limit: int = 80) -> list[dict[str, Any]]:
+    result = _git_stdout(
+        repo,
+        [
+            "log",
+            "--all",
+            "--topo-order",
+            "--date=iso-strict",
+            f"-n{limit}",
+            "--decorate=short",
+            "--pretty=format:%H%x1f%h%x1f%P%x1f%ad%x1f%an%x1f%s%x1f%b%x1f%D",
+        ],
+    )
+    commits: list[dict[str, Any]] = []
+    for line in result.splitlines():
+        if not line.strip():
+            continue
+        sha, short_sha, parents_raw, authored_at, author_name, subject, body, decorations = (
+            line.split("\x1f") + ["", "", "", "", "", "", "", ""]
+        )[:8]
+        commits.append(
+            {
+                "sha": sha.strip(),
+                "short_sha": short_sha.strip(),
+                "parents": [item for item in parents_raw.strip().split() if item],
+                "authored_at": authored_at.strip(),
+                "author_name": author_name.strip(),
+                "subject": subject.strip(),
+                "body": body.strip(),
+                "decorations": decorations.strip(),
+            }
+        )
+    return commits
+
+
+def _normalize_branch_refs(raw: Any) -> list[str]:
+    text = str(raw or "").strip()
+    if not text:
+        return []
+    refs: list[str] = []
+    for part in text.split(","):
+        cleaned = part.strip()
+        if not cleaned:
+            continue
+        if cleaned.startswith("HEAD -> "):
+            cleaned = cleaned[len("HEAD -> ") :].strip()
+        if cleaned.startswith("tag: "):
+            continue
+        if cleaned.startswith("origin/"):
+            continue
+        if cleaned == "HEAD":
+            continue
+        refs.append(cleaned)
+    return refs
+
+
+def _body_preview(body: str, *, max_lines: int = 3, max_chars: int = 220) -> str | None:
+    if not body:
+        return None
+    lines = [line.strip() for line in body.splitlines() if line.strip()]
+    if not lines:
+        return None
+    preview = " ".join(lines[:max_lines]).strip()
+    if len(preview) > max_chars:
+        preview = preview[: max_chars - 1].rstrip() + "…"
+    return preview or None
+
+
 def _normalize_patch_lines(patch: str) -> list[str]:
     lines = [line.rstrip("\n") for line in patch.splitlines()]
     if not lines:

package/src/deepscientist/gitops/service.py

@@ -100,7 +100,10 @@ def checkpoint_repo(repo: Path, message: str, allow_empty: bool = False) -> dict
             "branch": current_branch(repo),
             "head": head_commit(repo),
         }
-    result = run_command(["git", "commit", "-m", message], cwd=repo, check=False)
+    command = ["git", "commit", "-m", message]
+    if allow_empty:
+        command.insert(2, "--allow-empty")
+    result = run_command(command, cwd=repo, check=False)
     return {
         "committed": result.returncode == 0,
         "branch": current_branch(repo),