jupytermind 0.3.0

package/src/ai_data_scientist/insight_engine.py

@@ -0,0 +1,180 @@
+"""Insight & evidence engine.
+
+Implements DES-AIDS-010 (REQ-AIDS-009/010/027, ADR-0003): validates that a
+candidate insight is backed by an actually-executed notebook cell whose
+output contains the cited value, embeds a structured evidence manifest in
+the insight markdown cell, and refuses to write anything when evidence is
+missing.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+
+import nbformat
+
+from ai_data_scientist.project_manager import ProjectHandle, enqueue_write
+
+_MANIFEST_FENCE = "```evidence\n{payload}\n```"
+
+
+class EvidenceMissingError(ValueError):
+    """Raised when no executed cell backs a candidate insight's evidence."""
+
+
+class CitedValueNotFoundError(ValueError):
+    """Raised when a cited-value pattern has no match in a result's output."""
+
+
+# @id CODE-AIDS-125
+# @implements REQ-AIDS-010
+# @design DES-AIDS-010
+class AmbiguousEvidenceError(EvidenceMissingError):
+    """Raised when more than one executed cell matches the same evidence.
+
+    GitHub #54: duplicate ``execution_count`` values (common after a kernel
+    restart or appending to a notebook in a new session) can make more than
+    one code cell match an insight's ``execution_count``/``cited_value``
+    pair. Silently resolving to whichever cell is encountered first risks
+    attributing an insight to the wrong evidence, so this is raised instead
+    (a subclass of ``EvidenceMissingError`` so callers that already treat
+    missing evidence as "withhold the insight" handle this the same way).
+    """
+
+
+# @id CODE-AIDS-047
+# @implements REQ-AIDS-039
+# @design DES-AIDS-027
+def extract_cited_value(result: dict, pattern: str) -> str:
+    """Extract the exact substring a caller should pass as ``cited_value``.
+
+    Searches ``result["output"]`` (the dict returned by
+    ``mcp_gateway.run_and_record``/``execute_cell``) for ``pattern`` and
+    returns its first capture group verbatim, or the whole match when
+    ``pattern`` defines no group. Raises ``CitedValueNotFoundError`` on no
+    match instead of returning a guessed or empty value, so callers never
+    hand-transcribe (and risk rounding/mistyping) a value for
+    ``record_insight``.
+    """
+    output = str(result.get("output", ""))
+    match = re.search(pattern, output)
+    if match is None:
+        raise CitedValueNotFoundError(
+            f"Pattern {pattern!r} did not match the result output; "
+            "no cited value could be extracted."
+        )
+    return match.group(1) if match.lastindex else match.group(0)
+
+
+def _cell_output_contains(cell, cited_value: str) -> bool:
+    for output in cell.get("outputs", []):
+        for value in output.get("data", {}).values():
+            if cited_value in str(value):
+                return True
+        # GitHub #29: execute_result/display_data outputs store their
+        # payload under "data", but print()-produced stream output stores
+        # it under "text" instead; a value genuinely printed by the
+        # executed cell is equally valid evidence.
+        if output.get("output_type") == "stream" and cited_value in str(output.get("text", "")):
+            return True
+    return False
+
+
+def _matching_evidence_cells(notebook, execution_count: int, cited_value: str) -> list:
+    """Return every code cell whose execution_count/output matches evidence."""
+    return [
+        cell
+        for cell in notebook.cells
+        if cell.get("cell_type") == "code"
+        and cell.get("execution_count") == execution_count
+        and _cell_output_contains(cell, cited_value)
+    ]
+
+
+def _find_evidence_cell(notebook, execution_count: int, cited_value: str):
+    """Return the single code cell matching this evidence, or ``None``.
+
+    GitHub #54 (CODE-AIDS-125, REQ-AIDS-010): raises
+    ``AmbiguousEvidenceError`` instead of silently returning the first
+    match when more than one code cell shares the same ``execution_count``
+    and both produced ``cited_value`` in their output (e.g. after a kernel
+    restart or appending to a notebook in a new session re-uses an
+    ``execution_count``); callers must not guess which cell is the real
+    evidentiary basis.
+    """
+    matches = _matching_evidence_cells(notebook, execution_count, cited_value)
+    if len(matches) > 1:
+        raise AmbiguousEvidenceError(
+            f"{len(matches)} executed cells share execution_count="
+            f"{execution_count!r} and an output containing {cited_value!r}; "
+            "the evidentiary cell is ambiguous."
+        )
+    return matches[0] if matches else None
+
+
+# @id CODE-AIDS-009
+# @implements REQ-AIDS-009
+# @design DES-AIDS-010
+# @id CODE-AIDS-010
+# @implements REQ-AIDS-010
+# @design DES-AIDS-010
+# @id CODE-AIDS-027
+# @implements REQ-AIDS-027
+# @design DES-AIDS-010
+def record_insight(
+    handle: ProjectHandle,
+    insight_text: str,
+    evidence_execution_count: int,
+    cited_value: str,
+    claim_type: str,
+    language: str = "en",
+) -> None:
+    """Append an evidence-backed insight markdown cell, or refuse to.
+
+    Verifies the executed evidentiary cell exists and actually produced
+    ``cited_value`` before writing anything; raises ``EvidenceMissingError``
+    (REQ-AIDS-010) without touching the notebook otherwise. Raises
+    ``AmbiguousEvidenceError`` (a subclass of ``EvidenceMissingError``)
+    instead of guessing when more than one executed cell matches the same
+    ``execution_count``/``cited_value`` pair (GitHub #54).
+    """
+    notebook = nbformat.read(handle.notebook_path, as_version=4)
+    try:
+        evidence_cell = _find_evidence_cell(notebook, evidence_execution_count, cited_value)
+    except AmbiguousEvidenceError as exc:
+        message = (
+            f"Insight '{insight_text}' の根拠セルが一意に決まらないため記録を保留しました"
+            f"(execution_count={evidence_execution_count}の実行済みセルが複数あり、"
+            "いずれも該当する出力を含みます)。"
+            if language == "ja"
+            else (f"Withheld insight '{insight_text}': evidence is ambiguous ({exc}).")
+        )
+        raise AmbiguousEvidenceError(message) from exc
+    if evidence_cell is None:
+        message = (
+            f"Insight '{insight_text}' に根拠となる実行済みセル(execution_count="
+            f"{evidence_execution_count})が見つからないため記録を保留しました。"
+            if language == "ja"
+            else (
+                f"Could not establish supporting evidence for insight "
+                f"'{insight_text}' (no executed cell with execution_count="
+                f"{evidence_execution_count} producing '{cited_value}'); withheld."
+            )
+        )
+        raise EvidenceMissingError(message)
+
+    manifest = json.dumps(
+        {
+            "execution_count": evidence_execution_count,
+            "cited_value": cited_value,
+            "claim_type": claim_type,
+        },
+        separators=(",", ":"),
+    )
+    source = f"{insight_text}\n\n{_MANIFEST_FENCE.format(payload=manifest)}"
+
+    def add_cell(nb):
+        nb.cells.append(nbformat.v4.new_markdown_cell(source))
+
+    enqueue_write(handle, add_cell)

package/src/ai_data_scientist/japanese_nlp.py

@@ -0,0 +1,43 @@
+"""Japanese NLP via the pinned GiNZA (`ja_ginza`) pipeline.
+
+Implements DES-AIDS-021 (REQ-AIDS-023, ADR-0006): runs the GiNZA Japanese
+NLP pipeline on Japanese text to perform the requested operation and
+reports the result.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import spacy
+
+_SUPPORTED_OPERATIONS = ("tokenize",)
+_nlp = None  # module-level lazily-loaded ja_ginza pipeline singleton
+
+
+def _get_pipeline():
+    global _nlp
+    if _nlp is None:
+        _nlp = spacy.load("ja_ginza")
+    return _nlp
+
+
+@dataclass(frozen=True)
+class JapaneseNLPResult:
+    tokens: list
+    pos_tags: list
+
+
+# @id CODE-AIDS-023
+# @implements REQ-AIDS-023
+# @design DES-AIDS-021
+def analyze_japanese_text(text: str, operation: str = "tokenize") -> JapaneseNLPResult:
+    """Tokenize and POS-tag ``text`` using the pinned ja_ginza pipeline."""
+    if operation not in _SUPPORTED_OPERATIONS:
+        raise ValueError(f"Unsupported Japanese NLP operation: {operation!r}")
+
+    doc = _get_pipeline()(text)
+    tokens = [token.text for token in doc]
+    pos_tags = [token.pos_ for token in doc]
+
+    return JapaneseNLPResult(tokens=tokens, pos_tags=pos_tags)

package/src/ai_data_scientist/jupyter_launcher.py

@@ -0,0 +1,137 @@
+"""Real Jupyter MCP runtime launcher.
+
+Implements the production-grade RuntimeLauncher (DES-AIDS-025) backing
+mcp_runtime.ensure_runtime: starts real JupyterLab and jupyter-mcp-server
+processes inside the project's managed Python environment, bound to
+127.0.0.1 with auto-selected free ports and random tokens, and health-checks
+them over real HTTP. Verified interactively against a live jupyter-mcp-server
+2.2.3 instance: JupyterLab requires ``--IdentityProvider.token``, and the
+streamable-http transport requires its own ``--mcp-token`` (distinct from
+JUPYTER_TOKEN) or it refuses to start.
+"""
+
+from __future__ import annotations
+
+import os
+import secrets
+import signal
+import socket
+import subprocess
+import sys
+from pathlib import Path
+
+import httpx
+
+from ai_data_scientist.mcp_runtime import RuntimeInfo
+
+_HEALTH_CHECK_TIMEOUT_S = 2.0
+
+
+def _pick_free_port() -> int:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.bind(("127.0.0.1", 0))
+        return sock.getsockname()[1]
+
+
+def _venv_executable(name: str) -> str:
+    """Resolve ``name`` next to the current Python interpreter's venv bin dir."""
+    candidate = Path(sys.executable).parent / name
+    return str(candidate) if candidate.exists() else name
+
+
+# @id CODE-AIDS-044
+# @implements REQ-AIDS-034, REQ-AIDS-038
+# @design DES-AIDS-025
+class JupyterLabMCPServerLauncher:
+    """Starts real JupyterLab + jupyter-mcp-server subprocesses.
+
+    Satisfies the mcp_runtime.RuntimeLauncher Protocol with the real process
+    and HTTP transport details confirmed by interactive verification against
+    jupyter-mcp-server 2.2.3's streamable-http transport.
+    """
+
+    def __init__(self, working_dir: Path | None = None) -> None:
+        self._working_dir = working_dir or Path.cwd()
+
+    def start_jupyter(self) -> tuple[int, int, str]:
+        port = _pick_free_port()
+        token = secrets.token_urlsafe(32)
+        process = subprocess.Popen(
+            [
+                _venv_executable("jupyter-lab"),
+                f"--port={port}",
+                f"--IdentityProvider.token={token}",
+                "--ip=127.0.0.1",
+                "--no-browser",
+                f"--ServerApp.root_dir={self._working_dir}",
+            ],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+        return process.pid, port, token
+
+    def start_mcp_server(self, jupyter_port: int, jupyter_token: str) -> tuple[int, int, str]:
+        port = _pick_free_port()
+        token = secrets.token_urlsafe(32)
+        env = {
+            "JUPYTER_URL": f"http://127.0.0.1:{jupyter_port}",
+            "JUPYTER_TOKEN": jupyter_token,
+        }
+        process = subprocess.Popen(
+            [
+                _venv_executable("jupyter-mcp-server"),
+                "start",
+                "--transport=streamable-http",
+                f"--port={port}",
+                "--host=127.0.0.1",
+                f"--mcp-token={token}",
+            ],
+            env={**os.environ, **env},
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+        return process.pid, port, token
+
+    def is_healthy(self, info: RuntimeInfo) -> bool:
+        try:
+            response = httpx.post(
+                f"http://127.0.0.1:{info.mcp_port}/mcp",
+                headers={
+                    "Authorization": f"Bearer {info.mcp_token}",
+                    "Accept": "application/json, text/event-stream",
+                    "Content-Type": "application/json",
+                },
+                json={
+                    "jsonrpc": "2.0",
+                    "id": 0,
+                    "method": "initialize",
+                    "params": {
+                        "protocolVersion": "2024-11-05",
+                        "capabilities": {},
+                        "clientInfo": {"name": "ai-data-scientist", "version": "0.1"},
+                    },
+                },
+                timeout=_HEALTH_CHECK_TIMEOUT_S,
+            )
+            return response.status_code == 200
+        except httpx.HTTPError:
+            return False
+
+    def terminate(self, pid: int) -> None:
+        try:
+            os.kill(pid, signal.SIGTERM)
+        except ProcessLookupError:
+            pass
+
+    # @id CODE-AIDS-049
+    # @implements REQ-AIDS-041
+    # @design DES-AIDS-029
+    def is_process_alive(self, pid: int) -> bool:
+        """Probe whether ``pid`` is still alive without sending a real signal."""
+        try:
+            os.kill(pid, 0)
+        except ProcessLookupError:
+            return False
+        except PermissionError:
+            return True
+        return True

package/src/ai_data_scientist/jupyter_mcp_client.py

@@ -0,0 +1,94 @@
+"""Concrete Jupyter MCP client implementation.
+
+Implements DES-AIDS-026: satisfies the mcp_gateway.MCPClient contract by
+communicating with the jupyter-mcp-server runtime started by mcp_runtime, so
+run_and_record can execute real code against a live Jupyter kernel without
+any caller-supplied client (REQ-AIDS-038).
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Protocol
+
+from ai_data_scientist.mcp_gateway import MCPUnavailableError
+from ai_data_scientist.mcp_runtime import (
+    DEFAULT_STARTUP_TIMEOUT_MS,
+    DEFAULT_STATE_PATH,
+    RuntimeInfo,
+    ensure_runtime,
+)
+
+
+class _Transport(Protocol):
+    def __call__(self, port: int, token: str, code: str) -> dict: ...
+
+
+# @id CODE-AIDS-042
+# @implements REQ-AIDS-038
+# @design DES-AIDS-026
+class JupyterMCPClient:
+    """MCPClient implementation backed by a running jupyter-mcp-server.
+
+    ``transport`` is injected (rather than hard-coding an HTTP/stdio library
+    call) so this class is unit-testable without a real jupyter-mcp-server,
+    mirroring the Protocol-based testability used elsewhere in this codebase.
+    Transport-level connection failures are reclassified as the same
+    MCPUnavailableError already defined by mcp_gateway (DES-AIDS-004), so
+    callers never see transport-specific exceptions.
+    """
+
+    def __init__(self, runtime_info: RuntimeInfo, transport: _Transport) -> None:
+        self._runtime_info = runtime_info
+        self._transport = transport
+
+    def execute(self, code: str) -> dict:
+        try:
+            return self._transport(self._runtime_info.mcp_port, self._runtime_info.mcp_token, code)
+        except OSError as exc:
+            raise MCPUnavailableError(
+                "Jupyter MCP server connection failed (Jupyter MCPサーバーへの接続に失敗しました)."
+            ) from exc
+
+
+# @id CODE-AIDS-043
+# @implements REQ-AIDS-038
+# @design DES-AIDS-026
+def default_client(
+    launcher,
+    transport: _Transport,
+    timeout_ms: int = DEFAULT_STARTUP_TIMEOUT_MS,
+    state_path: Path = DEFAULT_STATE_PATH,
+) -> JupyterMCPClient:
+    """Ensure a runtime is running and return a client wired to it.
+
+    This is the factory mcp_gateway.run_and_record uses when no caller
+    supplies an explicit MCPClient, fulfilling REQ-AIDS-038's acceptance
+    criterion end-to-end.
+    """
+    runtime_info = ensure_runtime(launcher, state_path=state_path, timeout_ms=timeout_ms)
+    return JupyterMCPClient(runtime_info, transport)
+
+
+# @id CODE-AIDS-046
+# @implements REQ-AIDS-038
+# @design DES-AIDS-026
+def real_client(
+    timeout_ms: int = DEFAULT_STARTUP_TIMEOUT_MS,
+    state_path: Path = DEFAULT_STATE_PATH,
+) -> JupyterMCPClient:
+    """Convenience factory wiring the real JupyterLab/jupyter-mcp-server stack.
+
+    This is what production callers use in place of ``default_client`` when
+    they want the genuine subprocess launcher and streamable-http transport
+    rather than a test double.
+    """
+    from ai_data_scientist.jupyter_launcher import JupyterLabMCPServerLauncher
+    from ai_data_scientist.mcp_transport import execute_code
+
+    return default_client(
+        JupyterLabMCPServerLauncher(),
+        execute_code,
+        timeout_ms=timeout_ms,
+        state_path=state_path,
+    )

package/src/ai_data_scientist/language_router.py

@@ -0,0 +1,28 @@
+"""Bilingual (Japanese/English) instruction language detection."""
+
+_JAPANESE_RANGES = (
+    (0x3040, 0x309F),  # Hiragana
+    (0x30A0, 0x30FF),  # Katakana
+    (0x4E00, 0x9FFF),  # CJK Unified Ideographs
+    (0x3400, 0x4DBF),  # CJK Extension A
+    (0xFF66, 0xFF9F),  # Halfwidth Katakana
+)
+
+
+def _is_japanese_char(ch: str) -> bool:
+    code = ord(ch)
+    return any(low <= code <= high for low, high in _JAPANESE_RANGES)
+
+
+# @id CODE-AIDS-001
+# @implements REQ-AIDS-001
+# @design DES-AIDS-002
+def detect_language(text: str) -> str:
+    """Detect whether ``text`` is Japanese ("ja") or English ("en").
+
+    Deterministic, offline heuristic: any Japanese-script character present
+    marks the instruction as Japanese; otherwise it is treated as English.
+    """
+    if any(_is_japanese_char(ch) for ch in text):
+        return "ja"
+    return "en"