jupytermind 0.3.0

package/src/ai_data_scientist/explainability.py

@@ -0,0 +1,340 @@
+"""Model explainability.
+
+Implements DES-AIDS-019 plus CHANGE-011's DES-AIDS-070/071/072: preserves the
+legacy feature-importance ranking by default, and adds opt-in signed local
+contributions and permutation importance. CODE-AIDS-106 through CODE-AIDS-110
+cover importance-kind labeling, the signed-contribution provider chain
+(native pred_contrib / SHAP / linear fallback), and permutation importance.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from importlib import import_module
+from typing import Any, Literal
+
+import numpy as np
+import pandas as pd
+from sklearn.inspection import permutation_importance
+
+
+@dataclass(frozen=True)
+class ExplainabilityResult:
+    feature_importances: dict[str, float]
+    ranking: list[str]
+    importance_kind: str = "unknown"
+    contribution_kind: str | None = None
+    signed_contributions: list[dict[str, float]] | None = None
+    baseline_values: list[float] | None = None
+    raw_predictions: list[float] | None = None
+    additivity_check: dict[str, float | bool | None] | None = None
+    scoring: str | None = None
+
+
+def _as_feature_frame(x: Any, feature_names: list[str]) -> pd.DataFrame:
+    if x is None:
+        raise ValueError("x is required for this explainability method.")
+    if isinstance(x, pd.DataFrame):
+        return x.loc[:, feature_names].copy()
+
+    values = np.asarray(x)
+    if values.ndim != 2 or values.shape[1] != len(feature_names):
+        raise ValueError("x must be a 2D array-like with one column per feature name.")
+    return pd.DataFrame(values, columns=feature_names)
+
+
+def _build_ranking(feature_importances: dict[str, float]) -> list[str]:
+    return sorted(feature_importances, key=feature_importances.get, reverse=True)
+
+
+def _build_feature_importance_map(
+    feature_names: list[str], raw_importances: np.ndarray
+) -> dict[str, float]:
+    return {name: float(value) for name, value in zip(feature_names, np.asarray(raw_importances))}
+
+
+# @id CODE-AIDS-128
+# @implements REQ-AIDS-082
+# @design DES-AIDS-070
+def _normalize_coefficient_importances(coef: Any) -> np.ndarray:
+    raw_importances = np.abs(np.asarray(coef, dtype=float))
+    if raw_importances.ndim == 2 and raw_importances.shape[0] > 1:
+        return raw_importances.mean(axis=0)
+    return raw_importances.reshape(-1)
+
+
+# @id CODE-AIDS-107
+# @implements REQ-AIDS-082
+# @design DES-AIDS-070
+def _compute_default_importance(
+    model: object, feature_names: list[str]
+) -> tuple[dict[str, float], list[str], str]:
+    """Return the legacy global-importance ranking and its explicit kind."""
+    if hasattr(model, "feature_importances_"):
+        raw_importances = np.asarray(model.feature_importances_, dtype=float)
+        importance_kind = "split"
+    elif hasattr(model, "coef_"):
+        raw_importances = _normalize_coefficient_importances(model.coef_)
+        importance_kind = "coefficient_magnitude"
+    else:
+        raise ValueError("Model exposes neither feature_importances_ nor coef_.")
+
+    feature_importances = _build_feature_importance_map(feature_names, raw_importances)
+    return feature_importances, _build_ranking(feature_importances), importance_kind
+
+
+# @id CODE-AIDS-110
+# @implements REQ-AIDS-083
+# @design DES-AIDS-071
+def _predict_raw_output(model: object, frame: pd.DataFrame) -> np.ndarray | None:
+    """Best-effort raw-output prediction for additivity checks."""
+    if hasattr(model, "decision_function"):
+        raw = np.asarray(model.decision_function(frame), dtype=float)
+        return raw.reshape(-1)
+
+    for kwargs in ({"raw_score": True}, {"output_margin": True}):
+        try:
+            raw = np.asarray(model.predict(frame, **kwargs), dtype=float)
+        except (TypeError, ValueError):
+            continue
+        if raw.ndim == 1:
+            return raw
+        if raw.ndim == 2 and raw.shape[1] == 1:
+            return raw.reshape(-1)
+
+    if hasattr(model, "predict") and not hasattr(model, "predict_proba"):
+        try:
+            raw = np.asarray(model.predict(frame), dtype=float)
+        except (TypeError, ValueError):
+            return None
+        if raw.ndim == 1:
+            return raw
+        if raw.ndim == 2 and raw.shape[1] == 1:
+            return raw.reshape(-1)
+
+    return None
+
+
+def _finalize_signed_result(
+    feature_names: list[str],
+    contribution_kind: str,
+    contributions: np.ndarray,
+    baseline_values: np.ndarray,
+    raw_predictions: np.ndarray | None,
+) -> ExplainabilityResult:
+    contributions = np.asarray(contributions, dtype=float)
+    baseline_values = np.asarray(baseline_values, dtype=float).reshape(-1)
+    reconstructed = baseline_values + contributions.sum(axis=1)
+
+    if raw_predictions is None:
+        raw_prediction_values = None
+        checked_against_model_output = False
+        max_abs_error = None
+        passed = None
+    else:
+        raw_predictions = np.asarray(raw_predictions, dtype=float).reshape(-1)
+        raw_prediction_values = raw_predictions.astype(float).tolist()
+        checked_against_model_output = True
+        max_abs_error = float(np.max(np.abs(reconstructed - raw_predictions)))
+        passed = bool(max_abs_error <= 1e-6)
+    signed_contributions = [
+        {name: float(value) for name, value in zip(feature_names, row)} for row in contributions
+    ]
+    feature_importances = _build_feature_importance_map(
+        feature_names, np.mean(np.abs(contributions), axis=0)
+    )
+
+    return ExplainabilityResult(
+        feature_importances=feature_importances,
+        ranking=_build_ranking(feature_importances),
+        importance_kind="mean_absolute_signed_contribution",
+        contribution_kind=contribution_kind,
+        signed_contributions=signed_contributions,
+        baseline_values=baseline_values.astype(float).tolist(),
+        raw_predictions=raw_prediction_values,
+        additivity_check={
+            "passed": passed,
+            "max_abs_error": max_abs_error,
+            "checked_against_model_output": checked_against_model_output,
+        },
+    )
+
+
+def _normalize_shap_output(
+    values: np.ndarray, base_values: np.ndarray, n_rows: int, n_features: int
+) -> tuple[np.ndarray, np.ndarray] | None:
+    values = np.asarray(values, dtype=float)
+    base_values = np.asarray(base_values, dtype=float)
+
+    if values.ndim == 3:
+        values = values[..., -1]
+    if values.ndim != 2 or values.shape != (n_rows, n_features):
+        return None
+
+    if base_values.ndim == 0:
+        baseline = np.full(n_rows, float(base_values))
+    elif base_values.ndim == 1:
+        baseline = base_values.reshape(-1)
+    elif base_values.ndim == 2:
+        baseline = base_values[:, -1].reshape(-1)
+    else:
+        return None
+
+    if baseline.shape[0] != n_rows:
+        return None
+    return values, baseline
+
+
+# @id CODE-AIDS-109
+# @implements REQ-AIDS-083
+# @design DES-AIDS-071
+def _compute_signed_contributions(
+    model: object, frame: pd.DataFrame, feature_names: list[str]
+) -> ExplainabilityResult:
+    """Return signed local contributions from the best available provider."""
+    try:
+        native = np.asarray(model.predict(frame, pred_contrib=True), dtype=float)
+    except (AttributeError, TypeError, ValueError):
+        native = None
+
+    if native is not None and native.ndim == 2 and native.shape[0] == len(frame):
+        if native.shape[1] == len(feature_names) + 1:
+            contributions = native[:, :-1]
+            baseline_values = native[:, -1]
+        elif native.shape[1] == len(feature_names):
+            contributions = native
+            baseline_values = np.zeros(native.shape[0], dtype=float)
+        else:
+            contributions = None
+            baseline_values = None
+
+        if contributions is not None and baseline_values is not None:
+            return _finalize_signed_result(
+                feature_names,
+                contribution_kind="pred_contrib",
+                contributions=contributions,
+                baseline_values=baseline_values,
+                raw_predictions=_predict_raw_output(model, frame),
+            )
+
+    try:
+        shap = import_module("shap")
+    except ModuleNotFoundError:
+        shap = None
+
+    if shap is not None:
+        try:
+            explanation = shap.Explainer(model, frame)(frame)
+        except (AttributeError, NotImplementedError, TypeError, ValueError):
+            explanation = None
+        if explanation is not None:
+            normalized = _normalize_shap_output(
+                explanation.values,
+                explanation.base_values,
+                n_rows=len(frame),
+                n_features=len(feature_names),
+            )
+            if normalized is not None:
+                values, baseline_values = normalized
+                return _finalize_signed_result(
+                    feature_names,
+                    contribution_kind="shap",
+                    contributions=values,
+                    baseline_values=baseline_values,
+                    raw_predictions=_predict_raw_output(model, frame),
+                )
+
+    if hasattr(model, "coef_") and hasattr(model, "intercept_"):
+        coef = np.asarray(model.coef_, dtype=float).reshape(-1)
+        if coef.shape[0] != len(feature_names):
+            raise ValueError("Linear contribution path requires one coefficient per feature.")
+        baseline_values = np.full(len(frame), float(np.asarray(model.intercept_).reshape(-1)[0]))
+        contributions = frame.to_numpy(dtype=float) * coef
+        return _finalize_signed_result(
+            feature_names,
+            contribution_kind="linear",
+            contributions=contributions,
+            baseline_values=baseline_values,
+            raw_predictions=_predict_raw_output(model, frame),
+        )
+
+    raise ValueError(
+        "Signed contributions are unavailable for this model. Install optional "
+        "`shap`, use a model with native pred_contrib support, or request "
+        '`method="permutation"` instead.'
+    )
+
+
+# @id CODE-AIDS-108
+# @implements REQ-AIDS-084
+# @design DES-AIDS-072
+def _compute_permutation_importance(
+    model: object,
+    frame: pd.DataFrame,
+    y: Any,
+    feature_names: list[str],
+    scoring: str | None,
+    n_repeats: int,
+    random_state: int,
+) -> ExplainabilityResult:
+    """Return permutation importance with explicit scoring metadata."""
+    if y is None:
+        raise ValueError("y is required when method='permutation'.")
+
+    result = permutation_importance(
+        model,
+        frame,
+        np.asarray(y),
+        scoring=scoring,
+        n_repeats=n_repeats,
+        random_state=random_state,
+    )
+    feature_importances = _build_feature_importance_map(feature_names, result.importances_mean)
+    return ExplainabilityResult(
+        feature_importances=feature_importances,
+        ranking=_build_ranking(feature_importances),
+        importance_kind="permutation",
+        scoring=scoring,
+    )
+
+
+# @id CODE-AIDS-106
+# @implements REQ-AIDS-021 REQ-AIDS-082 REQ-AIDS-083 REQ-AIDS-084
+# @design DES-AIDS-019 DES-AIDS-070 DES-AIDS-071 DES-AIDS-072
+def explain_model(
+    model: object,
+    feature_names: list[str],
+    *,
+    method: Literal["default", "signed_contributions", "permutation"] = "default",
+    x: Any = None,
+    y: Any = None,
+    scoring: str | None = None,
+    n_repeats: int = 5,
+    random_state: int = 42,
+) -> ExplainabilityResult:
+    """Explain ``model`` with legacy ranking, signed contributions, or permutation importance."""
+    if method == "default":
+        feature_importances, ranking, importance_kind = _compute_default_importance(
+            model, feature_names
+        )
+        return ExplainabilityResult(
+            feature_importances=feature_importances,
+            ranking=ranking,
+            importance_kind=importance_kind,
+        )
+
+    frame = _as_feature_frame(x, feature_names)
+    if method == "signed_contributions":
+        return _compute_signed_contributions(model, frame, feature_names)
+    if method == "permutation":
+        return _compute_permutation_importance(
+            model=model,
+            frame=frame,
+            y=y,
+            feature_names=feature_names,
+            scoring=scoring,
+            n_repeats=n_repeats,
+            random_state=random_state,
+        )
+
+    raise ValueError(f"Unsupported explainability method: {method!r}")

package/src/ai_data_scientist/feature_engineering.py

@@ -0,0 +1,163 @@
+"""Feature engineering operations.
+
+Implements DES-AIDS-013 (REQ-AIDS-015): applies the requested encoding,
+scaling, group-wise aggregation, categorical interaction, missing-value
+flagging, or explicit-edge binning transformation to a dataframe and reports
+the resulting feature set.
+
+Also implements DES-AIDS-061 (REQ-AIDS-073): a leakage-safe fit/transform
+API that separates statistics estimation (``fit_features``) from
+transformation application (``transform_features``), so a cross-validation
+caller can fit on a training fold and transform a disjoint fold without ever
+deriving statistics from the held-out rows.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import pandas as pd
+from sklearn.preprocessing import StandardScaler
+
+_SUPPORTED_OPERATIONS = (
+    "one_hot",
+    "scale",
+    "aggregate",
+    "interaction",
+    "missing_flag",
+    "bin",
+)
+_SUPPORTED_AGG_FUNCS = ("mean", "sum", "count_eq")
+
+
+@dataclass(frozen=True)
+class FeatureResult:
+    dataframe: pd.DataFrame
+    added_columns: list[str]
+    removed_columns: list[str]
+    definitions: dict = field(default_factory=dict)
+
+
+# @id CODE-AIDS-093
+# @implements REQ-AIDS-073
+# @design DES-AIDS-061
+@dataclass(frozen=True)
+class FittedFeatureState:
+    operation: str
+    columns: tuple
+    scaler: StandardScaler
+
+
+def _require_params(operation: str, params: dict, required: list) -> None:
+    missing = [name for name in required if name not in params or params[name] is None]
+    if missing:
+        raise ValueError(f"Missing required params for operation {operation!r}: {missing}")
+
+
+# @id CODE-AIDS-015
+# @implements REQ-AIDS-015
+# @design DES-AIDS-013
+def engineer_features(
+    df: pd.DataFrame, operation: str, columns: list[str] | None = None, **params
+) -> FeatureResult:
+    """Apply ``operation`` (e.g. one-hot encoding, scaling) to ``df``."""
+    if operation not in _SUPPORTED_OPERATIONS:
+        raise ValueError(f"Unsupported feature engineering operation: {operation!r}")
+
+    target_columns = columns or list(df.columns)
+    definitions: dict = {}
+
+    if operation == "one_hot":
+        result_df = pd.get_dummies(df, columns=target_columns)
+        added_columns = [c for c in result_df.columns if c not in df.columns]
+        removed_columns = [c for c in target_columns if c not in result_df.columns]
+        for col in added_columns:
+            definitions[col] = f"one_hot encoding of {', '.join(target_columns)}"
+    elif operation == "scale":
+        result_df = df.copy()
+        scaler = StandardScaler()
+        result_df[target_columns] = scaler.fit_transform(df[target_columns])
+        added_columns = []
+        removed_columns = []
+    elif operation == "aggregate":
+        _require_params(operation, params, ["group_col", "agg_func"])
+        group_col = params["group_col"]
+        agg_func = params["agg_func"]
+        if agg_func not in _SUPPORTED_AGG_FUNCS:
+            raise ValueError(f"Unsupported agg_func: {agg_func!r}")
+        if agg_func == "count_eq":
+            _require_params(operation, params, ["compare_value"])
+        result_df = df.copy()
+        added_columns = []
+        for source_col in target_columns:
+            new_col = f"{source_col}_{agg_func}_by_{group_col}"
+            if agg_func == "count_eq":
+                compare_value = params["compare_value"]
+                result_df[new_col] = (
+                    df[source_col].eq(compare_value).groupby(df[group_col]).transform("sum")
+                )
+            else:
+                result_df[new_col] = df.groupby(group_col)[source_col].transform(agg_func)
+            definitions[new_col] = f"aggregate({agg_func}) of {source_col} grouped by {group_col}"
+            added_columns.append(new_col)
+        removed_columns = []
+    elif operation == "interaction":
+        _require_params(operation, params, ["col_a", "col_b"])
+        col_a = params["col_a"]
+        col_b = params["col_b"]
+        result_df = df.copy()
+        new_col = f"{col_a}__{col_b}_interaction"
+        result_df[new_col] = df[col_a].astype("string") + "__" + df[col_b].astype("string")
+        definitions[new_col] = f"interaction of {col_a} and {col_b}"
+        added_columns = [new_col]
+        removed_columns = []
+    elif operation == "missing_flag":
+        result_df = df.copy()
+        added_columns = []
+        for col in target_columns:
+            new_col = f"{col}_missing_flag"
+            result_df[new_col] = df[col].isna()
+            definitions[new_col] = f"missing_flag of {col}"
+            added_columns.append(new_col)
+        removed_columns = []
+    else:  # bin
+        _require_params(operation, params, ["edges"])
+        edges = params["edges"]
+        result_df = df.copy()
+        added_columns = []
+        for col in target_columns:
+            new_col = f"{col}_bin"
+            result_df[new_col] = pd.cut(df[col], bins=edges, right=True, include_lowest=True)
+            definitions[new_col] = f"bin of {col} with edges {edges}"
+            added_columns.append(new_col)
+        removed_columns = []
+
+    return FeatureResult(
+        dataframe=result_df,
+        added_columns=added_columns,
+        removed_columns=removed_columns,
+        definitions=definitions,
+    )
+
+
+# @id CODE-AIDS-094
+# @implements REQ-AIDS-073
+# @design DES-AIDS-061
+def fit_features(df: pd.DataFrame, operation: str, columns: list[str]) -> FittedFeatureState:
+    """Estimate statistics for ``operation`` from ``df`` only (no leakage)."""
+    if operation != "scale":
+        raise ValueError(f"Unsupported fit_features operation: {operation!r}")
+    scaler = StandardScaler()
+    scaler.fit(df[columns])
+    return FittedFeatureState(operation=operation, columns=tuple(columns), scaler=scaler)
+
+
+# @id CODE-AIDS-095
+# @implements REQ-AIDS-073
+# @design DES-AIDS-061
+def transform_features(fitted_state: FittedFeatureState, df: pd.DataFrame) -> FeatureResult:
+    """Apply ``fitted_state``'s already-estimated statistics to ``df``."""
+    columns = list(fitted_state.columns)
+    result_df = df.copy()
+    result_df[columns] = fitted_state.scaler.transform(df[columns])
+    return FeatureResult(dataframe=result_df, added_columns=[], removed_columns=[], definitions={})

package/src/ai_data_scientist/gate_config.py

@@ -0,0 +1,32 @@
+"""TDD verification gate configuration.
+
+Implements DES-AIDS-011 (REQ-AIDS-013): reads the musubix3 project
+configuration to confirm the required pytest command the gate depends on
+is correctly declared, so the gate can enforce a zero-failure pytest suite
+before any implementation change is considered complete.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+
+class TestCommandMissingError(ValueError):
+    """Raised when no required "test" command is declared in the config."""
+
+
+# @id CODE-AIDS-013
+# @implements REQ-AIDS-013
+# @design DES-AIDS-011
+def get_required_test_command(config_path: str = ".musubix/config.json") -> dict:
+    """Return the required "test" command entry from ``config_path``."""
+    config = json.loads(Path(config_path).read_text(encoding="utf-8"))
+    for command in config.get("commands", []):
+        if command.get("name") == "test":
+            if not command.get("required"):
+                raise TestCommandMissingError(
+                    "The 'test' command is declared but not marked required."
+                )
+            return command
+    raise TestCommandMissingError("No required 'test' command declared in config.")

package/src/ai_data_scientist/ingestion.py

@@ -0,0 +1,127 @@
+"""Data source ingestion.
+
+Implements DES-AIDS-005: loads CSV/Excel/database/API sources into an
+in-memory dataframe. A network allowlist is enforced before any remote
+call; a row-count limit is applied to the resulting dataframe after a
+remote (database/API) fetch completes, bounding downstream processing —
+not the fetch itself — and never applies to local CSV/Excel sources
+(REQ-AIDS-014, REQ-AIDS-032; GitHub #57).
+"""
+
+from __future__ import annotations
+
+import csv
+from dataclasses import dataclass
+from urllib.parse import urlparse
+
+import pandas as pd
+
+DEFAULT_ROW_LIMIT = 100_000
+_REMOTE_KINDS = ("api", "database")
+
+
+class NetworkAllowlistError(ValueError):
+    """Raised when a remote source host is not in the configured allowlist."""
+
+
+@dataclass(frozen=True)
+class SourceSpec:
+    """Describes a single ingestion request."""
+
+    kind: str  # "csv" | "excel" | "api" | "database"
+    location: str
+
+
+@dataclass(frozen=True)
+class IngestionResult:
+    dataframe: pd.DataFrame
+    row_count: int
+    column_count: int
+    truncated: bool
+    warnings: tuple[str, ...] = ()
+
+
+# @id CODE-AIDS-088
+# @implements REQ-AIDS-068
+# @design DES-AIDS-056
+def _sniff_csv_delimiter(path: str) -> tuple[str | None, bool]:
+    """Return ``(delimiter, sniff_succeeded)`` for the CSV file at ``path``.
+
+    Reads a bounded text sample and attempts ``csv.Sniffer().sniff`` over
+    comma/tab/semicolon candidates (DES-AIDS-056); on ``csv.Error`` (an
+    ambiguous sample), returns ``(None, False)`` so the caller falls back to
+    the existing comma-default behavior.
+    """
+    with open(path, encoding="utf-8", errors="replace") as handle:
+        sample = handle.read(8192)
+    try:
+        dialect = csv.Sniffer().sniff(sample, delimiters=",\t;")
+    except csv.Error:
+        return None, False
+    return dialect.delimiter, True
+
+
+# @id CODE-AIDS-014
+# @implements REQ-AIDS-014
+# @design DES-AIDS-005
+# @id CODE-AIDS-032
+# @implements REQ-AIDS-032
+# @design DES-AIDS-005
+def ingest(
+    source_spec: SourceSpec,
+    fetcher=None,
+    allowlist: tuple[str, ...] = (),
+    row_limit: int = DEFAULT_ROW_LIMIT,
+) -> IngestionResult:
+    """Load ``source_spec`` into a dataframe, applying ingestion safety rules.
+
+    For remote ``kind`` values ("api"/"database") the host is checked
+    against ``allowlist`` *before* ``fetcher`` is invoked, and the resulting
+    dataframe is truncated to ``row_limit`` rows if it exceeds that limit.
+    Local ``kind`` values ("csv"/"excel") are never truncated by
+    ``row_limit``: that limit is a remote-source safety control
+    (REQ-AIDS-032), not a general ingestion cap, so a local file is always
+    loaded in full (GitHub #57).
+    """
+    warnings: tuple[str, ...] = ()
+    if source_spec.kind == "csv":
+        delimiter, sniffed = _sniff_csv_delimiter(source_spec.location)
+        dataframe = pd.read_csv(source_spec.location, sep=delimiter if sniffed else ",")
+        if (
+            not sniffed
+            and len(dataframe.columns) == 1
+            and ("\t" in dataframe.columns[0] or ";" in dataframe.columns[0])
+        ):
+            warnings = (
+                f"CSV was parsed with the comma fallback as a single column named "
+                f"{dataframe.columns[0]!r}; the file may actually use a tab or "
+                "semicolon delimiter instead.",
+            )
+    elif source_spec.kind == "excel":
+        dataframe = pd.read_excel(source_spec.location)
+    elif source_spec.kind in _REMOTE_KINDS:
+        host = urlparse(source_spec.location).hostname
+        if host not in allowlist:
+            raise NetworkAllowlistError(
+                f"Host '{host}' is not in the configured allowlist "
+                f"(許可されていない接続先ホストです): {source_spec.location}"
+            )
+        if fetcher is None:
+            raise ValueError("A fetcher callable is required for remote ingestion sources.")
+        dataframe = fetcher(source_spec.location)
+    else:
+        raise ValueError(f"Unsupported ingestion source kind: {source_spec.kind!r}")
+
+    # GitHub #57: row_limit is a remote-source safety control (REQ-AIDS-032);
+    # local csv/excel files must never be silently truncated by it.
+    truncated = source_spec.kind in _REMOTE_KINDS and len(dataframe) > row_limit
+    if truncated:
+        dataframe = dataframe.iloc[:row_limit]
+
+    return IngestionResult(
+        dataframe=dataframe,
+        row_count=len(dataframe),
+        column_count=len(dataframe.columns),
+        truncated=truncated,
+        warnings=warnings,
+    )