jupytermind 0.3.0

package/src/ai_chemistry_scientist/structure_format_conversion.py

@@ -0,0 +1,84 @@
+"""Chemical structure format conversion module (DES-ACHEM-110 / REQ-ACHEM-110)."""
+
+from __future__ import annotations
+
+from rdkit import Chem
+from rdkit.Chem import inchi as rdkit_inchi
+
+from ai_chemistry_scientist.validation import fail, ok, register_validator
+
+_MODULE_NAME = "structure-format-conversion"
+
+INPUT_FORMATS = ("smiles", "inchi", "molblock")
+OUTPUT_FORMATS = ("smiles", "inchi", "inchikey", "molblock")
+
+_PARSERS = {
+    "smiles": Chem.MolFromSmiles,
+    "inchi": rdkit_inchi.MolFromInchi,
+    "molblock": Chem.MolFromMolBlock,
+}
+_WRITERS = {
+    "smiles": Chem.MolToSmiles,
+    "inchi": rdkit_inchi.MolToInchi,
+    "inchikey": rdkit_inchi.MolToInchiKey,
+    "molblock": Chem.MolToMolBlock,
+}
+
+
+def _parse_structure(input_format: str, input_value):
+    """Parse ``input_value`` with the ``input_format``-matching RDKit parser.
+
+    Rejects a non-str, empty, or unparseable value; a value that parses to a
+    zero-atom molecule (e.g. an empty SMILES or a ``0 0`` atom/bond-count
+    Molblock, which RDKit parses without error); and (same chemical-validity
+    domain as REQ-ACHEM-100 and every other module in this skill) any value
+    that parses but contains a dummy/query atom (RDKit atomic number 0).
+    """
+    if not isinstance(input_value, str) or not input_value:
+        return None
+    mol = _PARSERS[input_format](input_value)
+    if mol is None or mol.GetNumAtoms() == 0:
+        return None
+    if any(atom.GetAtomicNum() == 0 for atom in mol.GetAtoms()):
+        return None
+    return mol
+
+
+# @id CODE-ACHEM-922
+# @implements REQ-ACHEM-003 REQ-ACHEM-110
+# @design DES-ACHEM-002
+def _structure_format_conversion_validator(params: dict) -> dict:
+    for name in ("input_format", "output_format", "input_value"):
+        if name not in params:
+            return fail(name, "is required")
+    input_format = params["input_format"]
+    output_format = params["output_format"]
+    if input_format not in INPUT_FORMATS:
+        return fail("input_format", "must be one of the supported formats")
+    if output_format not in OUTPUT_FORMATS:
+        return fail("output_format", "must be one of the supported formats")
+    if _parse_structure(input_format, params["input_value"]) is None:
+        return fail(
+            "input_value",
+            f"must parse with the {input_format}-matching RDKit parser",
+        )
+    return ok()
+
+
+register_validator(_MODULE_NAME, _structure_format_conversion_validator)
+
+
+# @id CODE-ACHEM-110
+# @implements REQ-ACHEM-110
+# @design DES-ACHEM-110
+def run_structure_conversion(input_format: str, input_value: str, output_format: str) -> dict:
+    """Parse ``input_value`` per ``input_format`` and render per ``output_format``."""
+    mol = _parse_structure(input_format, input_value)
+    if mol is None or mol.GetNumAtoms() == 0:
+        raise ValueError("input_value must already be validated by the handler wrapper")
+
+    output_value = _WRITERS[output_format](mol)
+    return {
+        "output_format": output_format,
+        "output_value": output_value,
+    }

package/src/ai_chemistry_scientist/validation.py

@@ -0,0 +1,70 @@
+"""Shared parameter & chemical-validity validator (DES-ACHEM-002 / REQ-ACHEM-003)."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+ValidationResult = dict
+ValidatorFn = Callable[[dict[str, Any]], ValidationResult]
+BatchItemValidatorFn = Callable[[dict[str, Any]], ValidationResult]
+
+_REGISTRY: dict[str, ValidatorFn] = {}
+_BATCH_ITEM_REGISTRY: dict[str, BatchItemValidatorFn] = {}
+
+
+def ok() -> ValidationResult:
+    return {"ok": True}
+
+
+def fail(parameter: str, constraint: str) -> ValidationResult:
+    return {"ok": False, "parameter": parameter, "constraint": constraint}
+
+
+# @id CODE-ACHEM-002
+# @implements REQ-ACHEM-003
+# @design DES-ACHEM-002
+def register_validator(module_name: str, validator: ValidatorFn) -> None:
+    """Register ``module_name``'s own documented atomic-validity validator."""
+    _REGISTRY[module_name] = validator
+
+
+# @id CODE-ACHEM-913
+# @implements REQ-ACHEM-003
+# @design DES-ACHEM-002
+def register_batch_item_validator(module_name: str, validator: BatchItemValidatorFn) -> None:
+    """Register ``module_name``'s own documented per-item validator."""
+    _BATCH_ITEM_REGISTRY[module_name] = validator
+
+
+# @id CODE-ACHEM-914
+# @implements REQ-ACHEM-003
+# @design DES-ACHEM-002
+def validate_parameters(module_name: str, params: dict[str, Any]) -> ValidationResult:
+    """Dispatch to ``module_name``'s registered atomic validator.
+
+    Must run to completion before any descriptor computation, model fit, or
+    similarity/score calculation for the module's whole run (REQ-ACHEM-003).
+    """
+    validator = _REGISTRY.get(module_name)
+    if validator is None:
+        return fail("module", f"no validator registered for module '{module_name}'")
+    if not isinstance(params, dict):
+        return fail("params", "must be a dict")
+    return validator(params)
+
+
+# @id CODE-ACHEM-915
+# @implements REQ-ACHEM-003
+# @design DES-ACHEM-002
+def validate_batch_item(module_name: str, item_params: dict[str, Any]) -> ValidationResult:
+    """Dispatch to ``module_name``'s registered per-item validator.
+
+    Invoked once per batch item (REQ-ACHEM-010's per-item granularity); an
+    invalid item is rejected without aborting computation of the rest of the
+    batch.
+    """
+    validator = _BATCH_ITEM_REGISTRY.get(module_name)
+    if validator is None:
+        return fail("module", f"no batch-item validator registered for module '{module_name}'")
+    return validator(item_params)

package/src/ai_data_scientist/analysis_assumptions.py

@@ -0,0 +1,121 @@
+"""Analysis-assumption and applicability manifest.
+
+Implements DES-AIDS-042 (REQ-AIDS-054): records conclusion-critical
+analytical choices (preprocessing, sampling, causal scope) with an
+explicit status, so a written caveat is never mistaken for a verified
+check, and surfaces unresolved risk before a conclusion is finalized.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+_VALID_ASSUMPTION_STATUSES = frozenset({"verified", "tested", "assumed", "rejected"})
+_VALID_CAUSAL_SCOPES = frozenset({"descriptive", "associational", "causal"})
+_TESTED_OR_VERIFIED = frozenset({"tested", "verified"})
+
+
+# @id CODE-AIDS-074
+# @implements REQ-AIDS-054
+# @design DES-AIDS-042
+@dataclass(frozen=True)
+class Assumption:
+    """A single analytical assumption and its verification status."""
+
+    id: str
+    statement: str
+    status: str
+    evidence_cell: int | None = None
+    impact_if_false: str | None = None
+    conclusion_critical: bool = False
+
+    def __post_init__(self) -> None:
+        if self.status not in _VALID_ASSUMPTION_STATUSES:
+            raise ValueError(
+                f"status must be one of {sorted(_VALID_ASSUMPTION_STATUSES)}, got {self.status!r}."
+            )
+
+
+@dataclass(frozen=True)
+class AssumptionFinding:
+    """A single applicability-check observation."""
+
+    code: str
+    severity: str  # "error" | "warning"
+    message: str
+    assumption_id: str | None = None
+
+
+@dataclass(frozen=True)
+class AnalysisAssumptionManifest:
+    """Scope, assumptions, and causal classification for one analysis."""
+
+    analysis_scope: dict
+    assumptions: tuple[Assumption, ...] = field(default_factory=tuple)
+    causal_scope: str = "descriptive"
+    sampling: dict | None = None
+
+    def __post_init__(self) -> None:
+        if self.causal_scope not in _VALID_CAUSAL_SCOPES:
+            raise ValueError(
+                f"causal_scope must be one of {sorted(_VALID_CAUSAL_SCOPES)}, "
+                f"got {self.causal_scope!r}."
+            )
+
+    def unresolved_risks(self) -> tuple[Assumption, ...]:
+        """Conclusion-critical assumptions whose status is assumed or rejected."""
+        return tuple(
+            assumption
+            for assumption in self.assumptions
+            if assumption.conclusion_critical and assumption.status in ("assumed", "rejected")
+        )
+
+
+# @id CODE-AIDS-075
+# @implements REQ-AIDS-054
+# @design DES-AIDS-042
+def check_manifest(manifest: AnalysisAssumptionManifest) -> tuple[AssumptionFinding, ...]:
+    """Validate ``manifest``, returning one finding per detected gap."""
+    findings: list[AssumptionFinding] = []
+
+    if manifest.causal_scope == "causal":
+        has_identification = any(
+            assumption.status in _TESTED_OR_VERIFIED for assumption in manifest.assumptions
+        )
+        if not has_identification:
+            findings.append(
+                AssumptionFinding(
+                    code="missing_causal_identification",
+                    severity="error",
+                    message=(
+                        "causal_scope is 'causal' but no assumption has status "
+                        "'tested' or 'verified' to support identification."
+                    ),
+                )
+            )
+
+    for assumption in manifest.unresolved_risks():
+        findings.append(
+            AssumptionFinding(
+                code="unresolved_conclusion_critical_assumption",
+                severity="warning",
+                message=(
+                    f"Conclusion-critical assumption {assumption.id!r} has status "
+                    f"{assumption.status!r}, not verified/tested."
+                ),
+                assumption_id=assumption.id,
+            )
+        )
+
+    if manifest.sampling is not None:
+        missing = {"n", "seed"} - manifest.sampling.keys()
+        if missing:
+            findings.append(
+                AssumptionFinding(
+                    code="incomplete_sampling_record",
+                    severity="error",
+                    message=f"sampling is missing required keys: {sorted(missing)}.",
+                )
+            )
+
+    return tuple(findings)

package/src/ai_data_scientist/anomaly_detection.py

@@ -0,0 +1,39 @@
+"""Anomaly / outlier detection.
+
+Implements DES-AIDS-015 (REQ-AIDS-017): flags anomalous records in a
+dataframe using the requested detection method and reports the count of
+flagged records.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import pandas as pd
+
+_SUPPORTED_METHODS = ("zscore",)
+
+
+@dataclass(frozen=True)
+class AnomalyResult:
+    flagged_indices: list
+    method: str
+
+
+# @id CODE-AIDS-017
+# @implements REQ-AIDS-017
+# @design DES-AIDS-015
+def detect_anomalies(
+    df: pd.DataFrame, column: str, method: str = "zscore", params: dict | None = None
+) -> AnomalyResult:
+    """Flag anomalous rows of ``df[column]`` using ``method``."""
+    if method not in _SUPPORTED_METHODS:
+        raise ValueError(f"Unsupported anomaly detection method: {method!r}")
+    params = params or {}
+    threshold = params.get("threshold", 3.0)
+
+    series = df[column]
+    z_scores = (series - series.mean()) / series.std(ddof=0)
+    flagged = series.index[z_scores.abs() > threshold].tolist()
+
+    return AnomalyResult(flagged_indices=flagged, method=method)

package/src/ai_data_scientist/automl.py

@@ -0,0 +1,109 @@
+"""Automated model selection (AutoML).
+
+Implements DES-AIDS-018 (REQ-AIDS-020): trains multiple candidate model
+types via the supervised modeling and tuning interfaces and reports a
+ranked comparison of their evaluation metrics.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from typing import Any
+
+import pandas as pd
+
+from ai_data_scientist.ml_modeling import (
+    _DEFAULT_SCORING,
+    MODEL_BUILDERS,
+    _metric_direction,
+    build_cv_splits,
+    train_model,
+)
+
+
+@dataclass(frozen=True)
+class AutoMLResult:
+    ranked_candidates: list
+    scoring: str | None = None
+    cv_splits: list[tuple[list, list]] | None = None
+
+
+# @id CODE-AIDS-124
+# @implements REQ-AIDS-078
+# @design DES-AIDS-065 DES-AIDS-066
+def build_candidate_estimators(
+    model_type: str, candidate_estimators: dict[str, object | Callable[..., object]] | None = None
+) -> dict[str, object | Callable[..., object] | None]:
+    """Build the AutoML candidate registry."""
+    candidates: dict[str, object | Callable[..., object] | None] = {
+        name: None for name in MODEL_BUILDERS[model_type]
+    }
+    if candidate_estimators:
+        candidates.update(candidate_estimators)
+    return candidates
+
+
+# @id CODE-AIDS-020
+# @implements REQ-AIDS-020
+# @design DES-AIDS-018
+def run_automl(
+    df: pd.DataFrame,
+    target: str,
+    model_type: str = "classification",
+    scoring: str | None = None,
+    cv_strategy: str | None = None,
+    n_splits: int = 5,
+    random_state: int = 42,
+    groups: str | Sequence[Any] | pd.Series | None = None,
+    cv_splits: Sequence[tuple[Sequence[Any], Sequence[Any]]] | None = None,
+    candidate_estimators: dict[str, object | Callable[..., object]] | None = None,
+) -> AutoMLResult:
+    """Train several candidate model types and rank them by metric."""
+    metric_name = scoring or _DEFAULT_SCORING[model_type]
+
+    # @id CODE-AIDS-100
+    # @implements REQ-AIDS-077 REQ-AIDS-078
+    # @design DES-AIDS-065
+    shared_cv_splits = build_cv_splits(
+        df=df,
+        target=target,
+        model_type=model_type,
+        cv_strategy=cv_strategy,
+        n_splits=n_splits,
+        random_state=random_state,
+        groups=groups,
+        cv_splits=cv_splits,
+    )
+    candidates = []
+    for model_name, estimator in build_candidate_estimators(
+        model_type, candidate_estimators
+    ).items():
+        result = train_model(
+            df,
+            target=target,
+            model_type=model_type,
+            model_name=model_name if estimator is None else next(iter(MODEL_BUILDERS[model_type])),
+            scoring=scoring,
+            cv_strategy=cv_strategy,
+            n_splits=n_splits,
+            random_state=random_state,
+            groups=groups,
+            cv_splits=shared_cv_splits,
+            estimator=estimator,
+        )
+        candidates.append(
+            {
+                "model_name": model_name,
+                "metric": result.metrics[metric_name],
+                "fold_scores": result.fold_scores,
+                "result": result,
+            }
+        )
+
+    ranked = sorted(
+        candidates,
+        key=lambda candidate: candidate["metric"],
+        reverse=_metric_direction(model_type, scoring),
+    )
+    return AutoMLResult(ranked_candidates=ranked, scoring=metric_name, cv_splits=shared_cv_splits)

package/src/ai_data_scientist/cleaning.py

@@ -0,0 +1,56 @@
+"""Data cleaning operations.
+
+Implements DES-AIDS-006 (REQ-AIDS-004): performs a requested cleaning
+operation and reports the row/column impact.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import pandas as pd
+
+_SUPPORTED_OPERATIONS = ("drop_duplicates", "drop_na", "fillna")
+
+
+@dataclass(frozen=True)
+class CleaningReport:
+    dataframe: pd.DataFrame
+    rows_before: int
+    rows_after: int
+    rows_removed: int
+    columns_affected: list
+
+
+# @id CODE-AIDS-004
+# @implements REQ-AIDS-004
+# @design DES-AIDS-006
+def clean_dataset(
+    df: pd.DataFrame, operation: str, columns: list | None = None, fill_value=None
+) -> CleaningReport:
+    """Apply ``operation`` to ``df`` and report its row/column impact."""
+    if operation not in _SUPPORTED_OPERATIONS:
+        raise ValueError(f"Unsupported cleaning operation: {operation!r}")
+
+    rows_before = len(df)
+    target_columns = columns or list(df.columns)
+
+    if operation == "drop_duplicates":
+        cleaned = df.drop_duplicates()
+        columns_affected = list(df.columns)
+    elif operation == "drop_na":
+        cleaned = df.dropna(subset=target_columns)
+        columns_affected = target_columns
+    else:  # fillna
+        cleaned = df.copy()
+        cleaned[target_columns] = cleaned[target_columns].fillna(fill_value)
+        columns_affected = target_columns
+
+    rows_after = len(cleaned)
+    return CleaningReport(
+        dataframe=cleaned,
+        rows_before=rows_before,
+        rows_after=rows_after,
+        rows_removed=rows_before - rows_after,
+        columns_affected=columns_affected,
+    )

package/src/ai_data_scientist/cli.py

@@ -0,0 +1,90 @@
+"""Command-line entrypoint for the ai-data-scientist skill.
+
+This is a thin diagnostic/bootstrap CLI invoked via the npm wrapper
+(`bin/ai-data-scientist.js`); it is not itself part of the SDD requirement
+set. Copilot invokes the skill's Python modules directly per
+`.github/skills/ai-data-scientist/SKILL.md`, not through this CLI.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib
+
+_MODULES = [
+    "ai_data_scientist.language_router",
+    "ai_data_scientist.project_manager",
+    "ai_data_scientist.mcp_gateway",
+    "ai_data_scientist.ingestion",
+    "ai_data_scientist.cleaning",
+    "ai_data_scientist.eda",
+    "ai_data_scientist.stats_analysis",
+    "ai_data_scientist.visualization",
+    "ai_data_scientist.insight_engine",
+    "ai_data_scientist.gate_config",
+    "ai_data_scientist.skill_packaging",
+]
+
+
+def doctor() -> int:
+    """Import every skill module to confirm the environment is ready."""
+    failures = []
+    for module_name in _MODULES:
+        try:
+            importlib.import_module(module_name)
+        except Exception as exc:  # noqa: BLE001 - report every import failure
+            failures.append(f"{module_name}: {exc}")
+
+    if failures:
+        print("NG: 以下のモジュールを読み込めませんでした / failed to import:")
+        for failure in failures:
+            print(f"  - {failure}")
+        return 1
+
+    print(f"OK: {len(_MODULES)} モジュールを正常に読み込みました / modules import cleanly.")
+    return 0
+
+
+def validate_notebook(path: str) -> int:
+    """Audit a notebook (read-only) and print findings; exit 1 if any error finding."""
+    from ai_data_scientist.notebook_audit import audit_notebook
+
+    report = audit_notebook(path)
+    print(f"Notebook: {report.path}")
+    print(f"nbformat_valid={report.nbformat_valid} ok={report.ok}")
+    print(
+        f"code_cells={report.code_cell_count} executed={report.executed_code_cell_count} "
+        f"insight_cells={report.insight_cell_count}"
+    )
+    if not report.findings:
+        print("OK: 問題は見つかりませんでした / no issues found.")
+        return 0
+
+    print("findings:")
+    for finding in report.findings:
+        location = f"cell[{finding.cell_index}]" if finding.cell_index is not None else "-"
+        print(f"  - [{finding.severity}] {location}: {finding.message}")
+    return 0 if report.ok else 1
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="ai-data-scientist")
+    subparsers = parser.add_subparsers(dest="command")
+    subparsers.add_parser("doctor", help="Verify the Python environment can import every module.")
+    validate_parser = subparsers.add_parser(
+        "validate-notebook", help="Audit a notebook's execution/evidence completeness (read-only)."
+    )
+    validate_parser.add_argument("path", help="Path to the .ipynb file to audit.")
+    args = parser.parse_args(argv)
+
+    if args.command in (None, "doctor"):
+        return doctor()
+    if args.command == "validate-notebook":
+        return validate_notebook(args.path)
+
+    parser.error(f"Unknown command: {args.command}")
+    return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/src/ai_data_scientist/clustering.py

@@ -0,0 +1,54 @@
+"""Clustering and dimensionality reduction.
+
+Implements DES-AIDS-014 (REQ-AIDS-016): fits the requested unsupervised
+model (clustering or dimensionality reduction) and reports cluster
+assignments or reduced component values.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import pandas as pd
+from sklearn.cluster import KMeans
+from sklearn.decomposition import PCA
+
+_SUPPORTED_METHODS = ("kmeans", "pca")
+
+
+@dataclass(frozen=True)
+class UnsupervisedResult:
+    labels_or_components: object
+    method: str
+    params: dict
+
+
+# @id CODE-AIDS-016
+# @implements REQ-AIDS-016
+# @design DES-AIDS-014
+def cluster_or_reduce(
+    df: pd.DataFrame, method: str = "kmeans", params: dict | None = None
+) -> UnsupervisedResult:
+    """Fit ``method`` (e.g. kmeans, pca) on ``df`` and report the result."""
+    if method not in _SUPPORTED_METHODS:
+        raise ValueError(f"Unsupported unsupervised method: {method!r}")
+    params = dict(params or {})
+
+    if method == "kmeans":
+        params.setdefault("n_clusters", 2)
+        params.setdefault("n_init", 10)
+        if params["n_clusters"] > len(df):
+            raise ValueError(
+                f"n_clusters ({params['n_clusters']}) must not exceed the "
+                f"number of rows ({len(df)})"
+            )
+        model = KMeans(**params)
+        labels_or_components = model.fit_predict(df).tolist()
+    else:  # pca
+        params.setdefault("n_components", min(2, df.shape[1]))
+        model = PCA(**params)
+        labels_or_components = model.fit_transform(df).tolist()
+
+    return UnsupervisedResult(
+        labels_or_components=labels_or_components, method=method, params=params
+    )

package/src/ai_data_scientist/dashboard.py

@@ -0,0 +1,33 @@
+"""Interactive dashboard rendering.
+
+Implements DES-AIDS-023 (REQ-AIDS-024): renders an interactive widget or
+chart embedded in the notebook output, extending the MVP's static chart
+rendering with an interactive HTML MIME bundle.
+"""
+
+from __future__ import annotations
+
+import nbformat
+import pandas as pd
+import plotly.express as px
+
+_SUPPORTED_KINDS = ("scatter", "line", "bar")
+
+
+# @id CODE-AIDS-024
+# @implements REQ-AIDS-024
+# @design DES-AIDS-023
+def render_dashboard(df: pd.DataFrame, spec: dict) -> nbformat.NotebookNode:
+    """Render ``df`` per ``spec`` as an interactive HTML output bundle."""
+    kind = spec.get("kind", "scatter")
+    if kind not in _SUPPORTED_KINDS:
+        raise ValueError(f"Unsupported dashboard chart kind: {kind!r}")
+
+    plot_fn = {"scatter": px.scatter, "line": px.line, "bar": px.bar}[kind]
+    figure = plot_fn(df, x=spec.get("x"), y=spec.get("y"))
+    html = figure.to_html(include_plotlyjs="cdn", full_html=False)
+
+    return nbformat.v4.new_output(
+        "display_data",
+        data={"text/html": html, "text/plain": "<interactive plotly dashboard>"},
+    )