jupytermind 0.3.0

package/src/ai_scientist/manuscript.py

@@ -0,0 +1,94 @@
+"""Manuscript-writing delegation and final rendering."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+
+from ai_scientist.evidence_registry import query_evidence, record_evidence
+from ai_scientist.language import detect_language
+from ai_scientist.latex_renderer import render_latex
+from ai_scientist.project_config import load_project_config
+from ai_scientist.project_handle import ResearchProjectHandle
+from ai_scientist.skill_invocation import SkillInvoker
+
+TECH_WRITER_DEPENDENCY = {"skillId": "tech-writer", "version": "0.3.0"}
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _research_evidence_manifest(handle: ResearchProjectHandle) -> list[dict]:
+    return [
+        {
+            "project": record.project,
+            "phase": record.phase,
+            "artifactPath": record.artifact_path,
+            "artifactKind": record.artifact_kind,
+            "createdAt": record.created_at,
+            "metadata": record.metadata,
+        }
+        for record in query_evidence(handle)
+        if record.phase
+        in {"research-planning", "literature-review", "experimental-design", "data-analysis"}
+    ]
+
+
+def _write_markdown_artifact(handle: ResearchProjectHandle, content: str) -> Path:
+    path = handle.manuscript_dir / "manuscript.md"
+    path.write_text(content, encoding="utf-8")
+    return path
+
+
+# @id CODE-AISCI-010
+# @implements REQ-AISCI-009
+# @design DES-AISCI-006
+# @id CODE-AISCI-011
+# @implements REQ-AISCI-010
+# @design DES-AISCI-007
+# @id CODE-AISCI-022
+# @implements REQ-AISCI-021, REQ-AISCI-022
+# @design DES-AISCI-018
+def write_manuscript(
+    handle: ResearchProjectHandle, instruction: str, invoker: SkillInvoker
+) -> dict:
+    """Delegate manuscript drafting to tech-writer and render final format."""
+    evidence_manifest = _research_evidence_manifest(handle)
+    payload = {
+        "project": {"name": handle.name, "root": str(handle.root)},
+        "instruction": instruction,
+        "evidenceManifest": evidence_manifest,
+    }
+    response = invoker.invoke(
+        TECH_WRITER_DEPENDENCY["skillId"],
+        TECH_WRITER_DEPENDENCY["version"],
+        "write",
+        payload,
+    )
+    markdown_path = _write_markdown_artifact(handle, response["content"])
+    language = detect_language(instruction)
+    config = load_project_config(handle)
+    manuscript_format = config.get("manuscriptFormat", "markdown")
+    final_path = markdown_path
+    artifact_kind = "markdown"
+    if manuscript_format == "latex":
+        final_path = handle.manuscript_dir / "manuscript.tex"
+        final_path.write_text(render_latex(markdown_path), encoding="utf-8")
+        artifact_kind = "latex"
+    record = record_evidence(
+        handle,
+        "manuscript-writing",
+        final_path,
+        artifact_kind,
+        _now(),
+        metadata={"language": language, "sourceMarkdownPath": str(markdown_path)},
+    )
+    return {
+        "phase": "manuscript-writing",
+        "artifact": {
+            "phase": record.phase,
+            "path": record.artifact_path,
+            "metadata": record.metadata,
+        },
+    }

package/src/ai_scientist/mcp_config.py

@@ -0,0 +1,76 @@
+"""MCP server configuration loading for ai_scientist."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+
+class McpConfigError(ValueError):
+    """Invalid MCP configuration."""
+
+
+@dataclass(frozen=True)
+class ManagedServerConfig:
+    name: str
+    mode: str
+    launch_command: str
+    endpoint_template: str
+    health_check_path: str
+
+
+@dataclass(frozen=True)
+class ExternalServerConfig:
+    name: str
+    mode: str
+    endpoint: str
+
+
+def _require(payload: dict, field: str, server_name: str) -> str:
+    value = payload.get(field)
+    if not value:
+        raise McpConfigError(f"{server_name}: missing required field '{field}'")
+    return value
+
+
+# @id CODE-AISCI-016
+# @implements REQ-AISCI-016
+# @design DES-AISCI-011
+def load_mcp_config(
+    config_path: Path | str,
+) -> dict[str, ManagedServerConfig | ExternalServerConfig]:
+    """Load and validate MCP server entries."""
+    payload = json.loads(Path(config_path).read_text(encoding="utf-8"))
+    servers = payload.get("servers", [])
+    validated: dict[str, ManagedServerConfig | ExternalServerConfig] = {}
+    for entry in servers:
+        name = _require(entry, "name", "<unnamed>")
+        if name in validated:
+            raise McpConfigError(f"{name}: duplicate server name")
+        mode = _require(entry, "mode", name)
+        if mode == "managed":
+            launch_command = _require(entry, "launchCommand", name)
+            endpoint_template = _require(entry, "endpointTemplate", name)
+            health_check_path = _require(entry, "healthCheckPath", name)
+            if "{port}" not in launch_command:
+                raise McpConfigError(f"{name}: launchCommand must contain '{{port}}'")
+            if "{port}" not in endpoint_template:
+                raise McpConfigError(f"{name}: endpointTemplate must contain '{{port}}'")
+            validated[name] = ManagedServerConfig(
+                name=name,
+                mode=mode,
+                launch_command=launch_command,
+                endpoint_template=endpoint_template,
+                health_check_path=health_check_path,
+            )
+            continue
+        if mode == "external":
+            validated[name] = ExternalServerConfig(
+                name=name,
+                mode=mode,
+                endpoint=_require(entry, "endpoint", name),
+            )
+            continue
+        raise McpConfigError(f"{name}: mode must be 'managed' or 'external'")
+    return validated

package/src/ai_scientist/mcp_external.py

@@ -0,0 +1,42 @@
+"""External MCP endpoint connector."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from urllib import error, request
+
+from ai_scientist.mcp_config import ExternalServerConfig
+
+
+@dataclass(frozen=True)
+class HttpMcpClient:
+    """Simple HTTP MCP client used by the tests and local runtime glue."""
+
+    endpoint: str
+
+    def call_tool(self, tool_name: str, args: dict) -> dict:
+        payload = json.dumps({"tool": tool_name, "args": args}).encode("utf-8")
+        req = request.Request(
+            url=f"{self.endpoint.rstrip('/')}/tool",
+            data=payload,
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        try:
+            with request.urlopen(req, timeout=3) as response:
+                return json.loads(response.read().decode("utf-8"))
+        except (error.URLError, ConnectionError, TimeoutError) as exc:
+            raise ConnectionError(str(exc)) from exc
+
+
+# @id CODE-AISCI-018
+# @implements REQ-AISCI-018
+# @design DES-AISCI-013
+def connect_external_server(entry: ExternalServerConfig) -> HttpMcpClient:
+    """Return a client without managing any process lifecycle."""
+    if not isinstance(entry, ExternalServerConfig):
+        raise ValueError(
+            f"connect_external_server requires an external-mode config entry, got {entry!r}"
+        )
+    return HttpMcpClient(entry.endpoint)

package/src/ai_scientist/mcp_failures.py

@@ -0,0 +1,23 @@
+"""Failure classification for ai_scientist MCP access."""
+
+from __future__ import annotations
+
+
+class MCPUnavailableError(RuntimeError):
+    """MCP server unavailability with explicit server/phase attribution."""
+
+    def __init__(self, server_name: str, phase: str, error: Exception | str):
+        self.server_name = server_name
+        self.phase = phase
+        detail = str(error)
+        super().__init__(f"MCP server '{server_name}' is unreachable for phase '{phase}': {detail}")
+
+
+# @id CODE-AISCI-019
+# @implements REQ-AISCI-019
+# @design DES-AISCI-014
+def classify_mcp_failure(
+    server_name: str, phase: str, error: Exception | str
+) -> MCPUnavailableError:
+    """Build the classified error the caller must report."""
+    return MCPUnavailableError(server_name, phase, error)

package/src/ai_scientist/mcp_gateway.py

@@ -0,0 +1,38 @@
+"""MCP gateway that attributes calls to phases."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+from ai_scientist.mcp_failures import MCPUnavailableError, classify_mcp_failure
+
+
+class McpClient(Protocol):
+    """Minimal domain-tool client contract."""
+
+    def call_tool(self, tool_name: str, args: dict) -> dict: ...
+
+
+@dataclass
+class McpGateway:
+    """Route all ai_scientist external tool access through configured clients."""
+
+    clients: dict[str, McpClient]
+
+    # @id CODE-AISCI-015
+    # @implements REQ-AISCI-015
+    # @design DES-AISCI-010
+    def call_tool(self, server_name: str, tool_name: str, args: dict, phase: str) -> dict:
+        """Call one MCP tool via the configured client or raise a classified error."""
+        try:
+            client = self.clients[server_name]
+        except KeyError as exc:
+            raise classify_mcp_failure(server_name, phase, exc) from exc
+        try:
+            return client.call_tool(tool_name, args)
+        except Exception as exc:  # noqa: BLE001 - classified into explicit domain error
+            raise classify_mcp_failure(server_name, phase, exc) from exc
+
+
+__all__ = ["McpGateway", "MCPUnavailableError"]

package/src/ai_scientist/mcp_managed.py

@@ -0,0 +1,180 @@
+"""Managed MCP process lifecycle for ai_scientist."""
+
+from __future__ import annotations
+
+import ipaddress
+import shlex
+import socket
+import subprocess
+import threading
+import time
+from dataclasses import dataclass
+from urllib import error, request
+from urllib.parse import urlparse
+
+from ai_scientist.mcp_config import ManagedServerConfig
+from ai_scientist.mcp_failures import classify_mcp_failure
+
+
+@dataclass(frozen=True)
+class RuntimeInfo:
+    """Runtime identity for one managed MCP server."""
+
+    pid: int
+    port: int
+    endpoint: str
+
+
+_RUNTIMES: dict[str, tuple[subprocess.Popen, RuntimeInfo, str]] = {}
+
+# Per-server-name locks, guarded by a single registry lock so lock creation
+# itself cannot race. Mirrors the ai_data_scientist project_manager per-path
+# lock pattern, but here the lock additionally protects the entire
+# check-then-start critical section in ensure_managed_server, so two
+# concurrent first-use callers for the same server name cannot both observe
+# "no running process" and both start a second process.
+_runtime_locks: dict[str, threading.Lock] = {}
+_runtime_locks_guard = threading.Lock()
+
+
+def _lock_for(name: str) -> threading.Lock:
+    with _runtime_locks_guard:
+        lock = _runtime_locks.get(name)
+        if lock is None:
+            lock = threading.Lock()
+            _runtime_locks[name] = lock
+        return lock
+
+
+def _is_loopback_host(hostname: str | None) -> bool:
+    """True when ``hostname`` is a loopback address, or a name (such as
+    ``localhost``) that resolves only to loopback addresses. Resolving
+    rather than string-matching ``localhost`` prevents a host/DNS override
+    that maps it to a non-loopback address from bypassing the check.
+    A name with no resolvable addresses is rejected, not treated as safe."""
+    if hostname is None:
+        return False
+    try:
+        return ipaddress.ip_address(hostname).is_loopback
+    except ValueError:
+        pass
+    try:
+        resolved = socket.getaddrinfo(hostname, None)
+    except OSError:
+        return False
+    return bool(resolved) and all(ipaddress.ip_address(info[4][0]).is_loopback for info in resolved)
+
+
+def _require_loopback_endpoint(entry: ManagedServerConfig) -> None:
+    """Reject a non-loopback ``endpointTemplate`` host before any port is
+    allocated or process started (DES-AISCI-012: "must bind only to
+    127.0.0.1"). Closes jupytermind#66 / CHANGE-019 (ADR-0084); see
+    TEST-AISCI-042.
+    """
+    sample_endpoint = entry.endpoint_template.format(port=0)
+    hostname = urlparse(sample_endpoint).hostname
+    if not _is_loopback_host(hostname):
+        raise classify_mcp_failure(
+            entry.name,
+            "managed-startup",
+            f"endpointTemplate host {hostname!r} is not loopback; "
+            "managed servers must bind only to 127.0.0.1",
+        )
+
+
+def _allocate_port() -> int:
+    with socket.socket() as sock:
+        sock.bind(("127.0.0.1", 0))
+        return int(sock.getsockname()[1])
+
+
+def _healthy(endpoint: str, health_check_path: str) -> bool:
+    try:
+        with request.urlopen(f"{endpoint}{health_check_path}", timeout=1) as response:
+            return response.status == 200
+    except (error.URLError, TimeoutError):
+        return False
+
+
+def _stop_process(process: subprocess.Popen) -> None:
+    if process.poll() is None:
+        process.terminate()
+        try:
+            process.wait(timeout=2)
+        except subprocess.TimeoutExpired:
+            process.kill()
+            process.wait(timeout=2)
+
+
+# @id CODE-AISCI-017
+# @implements REQ-AISCI-017
+# @design DES-AISCI-012
+# @id CODE-AISCI-027
+# @implements REQ-AISCI-019
+# @design DES-AISCI-014
+def ensure_managed_server(
+    entry: ManagedServerConfig,
+    timeout_ms: int = 30000,
+    phase: str = "managed-startup",
+) -> RuntimeInfo:
+    """Start a managed server on first use and reuse it for the session."""
+    _require_loopback_endpoint(entry)
+    with _lock_for(entry.name):
+        existing = _RUNTIMES.get(entry.name)
+        if existing is not None:
+            process, info, health_check_path = existing
+            if process.poll() is None and _healthy(info.endpoint, health_check_path):
+                return info
+            _stop_locked(entry.name)
+
+        port = _allocate_port()
+        endpoint = entry.endpoint_template.format(port=port)
+        command = shlex.split(entry.launch_command.format(port=port))
+        process = subprocess.Popen(
+            command,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+        info = RuntimeInfo(pid=process.pid, port=port, endpoint=endpoint)
+        deadline = time.monotonic() + timeout_ms / 1000
+        while time.monotonic() < deadline:
+            # Check our own child is still alive *before* trusting a health
+            # response: an external process that stole the allocated port
+            # could otherwise answer the health probe on behalf of our dead
+            # child process and get cached as a legitimate managed runtime.
+            if process.poll() is not None:
+                break
+            if _healthy(endpoint, entry.health_check_path):
+                _RUNTIMES[entry.name] = (process, info, entry.health_check_path)
+                return info
+            time.sleep(0.1)
+        _stop_process(process)
+        raise classify_mcp_failure(entry.name, phase, "startup timeout")
+
+
+def status(server_name: str) -> RuntimeInfo | None:
+    """Return current runtime info when the managed server is still running."""
+    with _lock_for(server_name):
+        existing = _RUNTIMES.get(server_name)
+        if existing is None:
+            return None
+        process, info, health_check_path = existing
+        if process.poll() is None and _healthy(info.endpoint, health_check_path):
+            return info
+        _stop_locked(server_name)
+        return None
+
+
+def stop(server_name: str) -> None:
+    """Terminate only the registered managed server for ``server_name``."""
+    with _lock_for(server_name):
+        _stop_locked(server_name)
+
+
+def _stop_locked(server_name: str) -> None:
+    """Terminate the registered server; caller must already hold its lock."""
+    existing = _RUNTIMES.pop(server_name, None)
+    if existing is None:
+        return
+    process, _info, _health_check_path = existing
+    _stop_process(process)

package/src/ai_scientist/npm_packaging.py

@@ -0,0 +1,49 @@
+"""Packaging helpers for ai-scientist npm bootstrap regressions."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from collections.abc import Sequence
+from pathlib import Path
+
+
+# @id CODE-AISCI-027
+# @implements REQ-AISCI-025 REQ-AGENOM-080 REQ-AIMS-080 REQ-ASTRUCT-060
+# @design DES-AISCI-020 DES-AGENOM-080 DES-AIMS-080 DES-ASTRUCT-060
+def load_package_files(package_json_path: str | Path = "package.json") -> list[str]:
+    """Return the npm package whitelist entries from package.json."""
+    package_json = Path(package_json_path)
+    return json.loads(package_json.read_text("utf-8"))["files"]
+
+
+# @id CODE-AISCI-028
+# @implements REQ-AISCI-025 REQ-AGENOM-080 REQ-AIMS-080 REQ-ASTRUCT-060
+# @design DES-AISCI-020 DES-AGENOM-080 DES-AIMS-080 DES-ASTRUCT-060
+def load_npm_pack_dry_run_paths(project_root: str | Path = ".") -> set[str]:
+    """Return packed file paths reported by `npm pack --dry-run --json`."""
+    result = subprocess.run(
+        ["npm", "pack", "--dry-run", "--json"],
+        check=True,
+        capture_output=True,
+        cwd=Path(project_root),
+        text=True,
+    )
+    payload = json.loads(result.stdout)
+    return {entry["path"] for pack in payload for entry in pack["files"]}
+
+
+# @id CODE-AISCI-029
+# @implements REQ-AISCI-025 REQ-AGENOM-080 REQ-AIMS-080 REQ-ASTRUCT-060
+# @design DES-AISCI-020 DES-AGENOM-080 DES-AIMS-080 DES-ASTRUCT-060
+def iter_skill_python_globs(package_files: Sequence[str]) -> dict[str, str]:
+    """Map exact shipped skill entries to expected Python source globs."""
+    mappings = {}
+    for entry in package_files:
+        if not entry.startswith(".github/skills/") or "/" in entry.removeprefix(".github/skills/"):
+            continue
+        slug = entry.removeprefix(".github/skills/")
+        package_dir = Path("src") / slug.replace("-", "_")
+        if (package_dir / "__init__.py").exists():
+            mappings[entry] = f"{package_dir.as_posix()}/**/*.py"
+    return mappings

package/src/ai_scientist/orchestrator.py

@@ -0,0 +1,133 @@
+"""Top-level ai_scientist dispatch orchestration."""
+
+from __future__ import annotations
+
+import importlib
+import inspect
+
+from ai_scientist.language import detect_language
+from ai_scientist.manifest import (
+    load_phase_manifest,
+    scan_repo_skill_registry,
+    verify_manifest_against_registry,
+)
+from ai_scientist.mcp_gateway import McpGateway
+from ai_scientist.phase_gate import check_gate
+from ai_scientist.phase_state import active_phase
+from ai_scientist.project_handle import resolve_research_project
+from ai_scientist.skill_invocation import DefaultSkillInvoker
+
+_PHASE_HINTS = (
+    ("research-planning", ("research-planning", "research plan", "研究計画")),
+    (
+        "literature-review",
+        ("literature-review", "literature review", "search the literature", "文献", "先行研究"),
+    ),
+    ("experimental-design", ("experimental-design", "experimental design", "実験設計")),
+    ("data-analysis", ("data-analysis", "analyze", "analyse", "データ分析", "分析して")),
+    ("manuscript-writing", ("manuscript-writing", "write the manuscript", "manuscript", "原稿")),
+    ("peer-review", ("peer-review", "review the manuscript", "査読")),
+    ("reproducibility-check", ("reproducibility-check", "reproducibility", "再現性")),
+    ("presentation", ("presentation", "slides", "発表", "スライド")),
+)
+
+
+def _message(language: str, phase: str) -> str:
+    if language == "ja":
+        return f"{phase} フェーズを処理しました。"
+    return f"Processed the {phase} phase."
+
+
+def _gateway_for(mcp_client):
+    if mcp_client is None:
+        return None
+    return McpGateway(clients={"default": mcp_client})
+
+
+# @id CODE-AISCI-025
+# @implements REQ-AISCI-001
+# @design DES-AISCI-001
+def _requested_phase_from_instruction(instruction: str) -> str | None:
+    normalized = instruction.casefold()
+    for phase, hints in _PHASE_HINTS:
+        if any(hint in normalized or hint in instruction for hint in hints):
+            return phase
+    return None
+
+
+# @id CODE-AISCI-026
+# @implements REQ-AISCI-023
+# @design DES-AISCI-015
+def _dispatch_phase(
+    phase: str,
+    manifest: dict,
+    handle,
+    instruction: str,
+    skill_invoker,
+    mcp_client,
+):
+    entry = manifest.get(phase)
+    if entry is None:
+        raise ValueError(f"Unknown phase: {phase}")
+    module = importlib.import_module(entry["modulePath"])
+    handler = getattr(module, entry["functionName"])
+    gateway = _gateway_for(mcp_client)
+    available_args = {
+        "handle": handle,
+        "instruction": instruction,
+        "invoker": skill_invoker,
+        "mcp_client": mcp_client,
+        "gateway": gateway,
+        "server_name": "default" if gateway is not None else None,
+    }
+    signature = inspect.signature(handler)
+    call_args = {
+        name: value
+        for name, value in available_args.items()
+        if name in signature.parameters and value is not None
+    }
+    return handler(**call_args)
+
+
+# @id CODE-AISCI-014
+# @implements REQ-AISCI-001
+# @design DES-AISCI-001
+def dispatch(
+    instruction: str,
+    project_name: str,
+    requested_phase: str | None = None,
+    override_reason: str | None = None,
+    projects_root=None,
+    skill_invoker=None,
+    mcp_client=None,
+) -> dict:
+    """Resolve the project, verify the manifest, gate phases, and dispatch."""
+    language = detect_language(instruction)
+    handle = resolve_research_project(project_name, projects_root=projects_root)
+    manifest = load_phase_manifest()
+    verify_manifest_against_registry(manifest, scan_repo_skill_registry())
+    current_active = active_phase(handle)
+    phase = requested_phase or _requested_phase_from_instruction(instruction) or current_active
+    phase = phase or "research-planning"
+    if phase != current_active:
+        decision = check_gate(
+            handle,
+            phase,
+            override={"reason": override_reason} if override_reason else None,
+        )
+        if not decision.allowed:
+            return {"language": language, "message": decision.message, "phase": phase}
+    result = _dispatch_phase(
+        phase,
+        manifest,
+        handle,
+        instruction,
+        skill_invoker or DefaultSkillInvoker(),
+        mcp_client,
+    )
+    return {
+        "language": language,
+        "message": _message(language, phase),
+        "phase": phase,
+        "result": result,
+    }