@nahisaho/satori 0.22.0 → 0.24.0

package/src/.github/skills/scientific-data-profiling/SKILL.md

@@ -0,0 +1,247 @@
+---
+name: scientific-data-profiling
+description: |
+  データプロファイリング・品質スキル。ydata-profiling 自動 EDA ・
+  Great Expectations データバリデーション・データ品質スコア・
+  型推論・相関検出・外れ値フラグ・データカタログ生成。
+---
+
+# Scientific Data Profiling
+
+データセットの包括的プロファイリング・品質評価・
+自動 EDA レポートパイプラインを提供する。
+
+## When to Use
+
+- 新しいデータセットの全体像を素早く把握するとき
+- データ品質スコアを算出して品質基準をチェックするとき
+- ydata-profiling で自動 EDA レポートを生成するとき
+- Great Expectations でデータバリデーションルールを定義するとき
+- データカタログ (辞書) を自動生成するとき
+- 相関・外れ値・欠損を一括診断するとき
+
+---
+
+## Quick Start
+
+## 1. ydata-profiling 自動 EDA
+
+```python
+import numpy as np
+import pandas as pd
+
+
+def auto_profile_report(df, title="Data Profile Report",
+                        minimal=False, output="profile_report.html"):
+    """
+    ydata-profiling 自動 EDA レポート。
+
+    Parameters:
+        df: pd.DataFrame — 入力データ
+        title: str — レポートタイトル
+        minimal: bool — 軽量モード
+        output: str — 出力 HTML パス
+    """
+    from ydata_profiling import ProfileReport
+
+    profile = ProfileReport(
+        df, title=title, minimal=minimal,
+        correlations={"pearson": {"calculate": True},
+                      "spearman": {"calculate": True},
+                      "kendall": {"calculate": True}},
+        missing_diagrams={"bar": True, "matrix": True, "heatmap": True})
+
+    profile.to_file(output)
+
+    # サマリー抽出
+    desc = profile.get_description()
+    summary = {
+        "n_rows": len(df),
+        "n_cols": len(df.columns),
+        "n_numeric": len(df.select_dtypes(include=[np.number]).columns),
+        "n_categorical": len(df.select_dtypes(include=["object", "category"]).columns),
+        "total_missing": int(df.isnull().sum().sum()),
+        "missing_pct": float(df.isnull().sum().sum() / (len(df) * len(df.columns)) * 100),
+        "n_duplicates": int(df.duplicated().sum()),
+    }
+
+    print(f"Profile Report → {output}")
+    print(f"  {summary['n_rows']} rows × {summary['n_cols']} cols, "
+          f"{summary['missing_pct']:.1f}% missing, "
+          f"{summary['n_duplicates']} duplicates")
+    return {"report_path": output, "summary": summary}
+```
+
+## 2. データ品質スコア
+
+```python
+def data_quality_score(df, rules=None):
+    """
+    データ品質スコア算出 (0-100)。
+
+    Parameters:
+        df: pd.DataFrame — 入力データ
+        rules: dict | None — カスタムルール
+    """
+    scores = {}
+
+    # 1. 完全性 (Completeness) — 非欠損率
+    completeness = 1.0 - df.isnull().sum().sum() / (len(df) * len(df.columns))
+    scores["completeness"] = completeness
+
+    # 2. 一意性 (Uniqueness) — 非重複率
+    uniqueness = 1.0 - df.duplicated().sum() / len(df) if len(df) > 0 else 1.0
+    scores["uniqueness"] = uniqueness
+
+    # 3. 一貫性 (Consistency) — 型一貫性
+    type_consistent = 0
+    for col in df.columns:
+        non_null = df[col].dropna()
+        if len(non_null) == 0:
+            type_consistent += 1
+            continue
+        try:
+            inferred = pd.api.types.infer_dtype(non_null, skipna=True)
+            if inferred not in ["mixed", "mixed-integer"]:
+                type_consistent += 1
+        except Exception:
+            pass
+    consistency = type_consistent / len(df.columns) if len(df.columns) > 0 else 1.0
+    scores["consistency"] = consistency
+
+    # 4. 適時性 (Timeliness) — 日付カラムの新しさ
+    date_cols = df.select_dtypes(include=["datetime64"]).columns
+    if len(date_cols) > 0:
+        max_date = df[date_cols[0]].max()
+        freshness = 1.0  # Placeholder
+        scores["timeliness"] = freshness
+    else:
+        scores["timeliness"] = 1.0
+
+    # 5. 妥当性 (Validity) — 数値カラムの有限性
+    numeric_cols = df.select_dtypes(include=[np.number]).columns
+    if len(numeric_cols) > 0:
+        finite_rate = df[numeric_cols].apply(lambda x: np.isfinite(x.dropna()).mean()).mean()
+        scores["validity"] = float(finite_rate)
+    else:
+        scores["validity"] = 1.0
+
+    # 総合スコア
+    weights = {"completeness": 0.3, "uniqueness": 0.2,
+               "consistency": 0.2, "timeliness": 0.1, "validity": 0.2}
+    total_score = sum(scores[k] * weights[k] for k in weights) * 100
+
+    # カスタムルール
+    rule_results = []
+    if rules:
+        for rule_name, rule_fn in rules.items():
+            try:
+                passed = rule_fn(df)
+                rule_results.append({"rule": rule_name, "passed": passed})
+            except Exception as e:
+                rule_results.append({"rule": rule_name, "passed": False,
+                                     "error": str(e)})
+
+    print(f"Data Quality Score: {total_score:.1f}/100")
+    for k, v in scores.items():
+        print(f"  {k}: {v:.3f}")
+
+    return {"total_score": total_score, "dimension_scores": scores,
+            "rule_results": rule_results}
+```
+
+## 3. Great Expectations バリデーション
+
+```python
+def great_expectations_validate(df, expectations=None):
+    """
+    Great Expectations スタイルのデータバリデーション。
+
+    Parameters:
+        df: pd.DataFrame — 入力データ
+        expectations: list[dict] | None — バリデーションルール
+    """
+    if expectations is None:
+        expectations = _auto_generate_expectations(df)
+
+    results = []
+    for exp in expectations:
+        exp_type = exp["type"]
+        col = exp.get("column")
+        kwargs = exp.get("kwargs", {})
+
+        try:
+            if exp_type == "expect_column_to_exist":
+                success = col in df.columns
+            elif exp_type == "expect_column_values_to_not_be_null":
+                max_pct = kwargs.get("mostly", 1.0)
+                non_null_pct = df[col].notnull().mean()
+                success = non_null_pct >= max_pct
+            elif exp_type == "expect_column_values_to_be_between":
+                min_val, max_val = kwargs["min_value"], kwargs["max_value"]
+                vals = df[col].dropna()
+                success = bool((vals >= min_val).all() and (vals <= max_val).all())
+            elif exp_type == "expect_column_values_to_be_unique":
+                success = not df[col].duplicated().any()
+            elif exp_type == "expect_column_values_to_be_in_set":
+                valid_set = set(kwargs["value_set"])
+                success = df[col].dropna().isin(valid_set).all()
+            elif exp_type == "expect_table_row_count_to_be_between":
+                success = kwargs["min_value"] <= len(df) <= kwargs["max_value"]
+            else:
+                success = None
+
+            results.append({"expectation": exp_type, "column": col,
+                            "success": success})
+        except Exception as e:
+            results.append({"expectation": exp_type, "column": col,
+                            "success": False, "error": str(e)})
+
+    results_df = pd.DataFrame(results)
+    n_pass = results_df["success"].sum()
+    n_total = len(results_df)
+    print(f"Validation: {n_pass}/{n_total} expectations passed "
+          f"({n_pass/n_total*100:.0f}%)")
+    return results_df
+
+
+def _auto_generate_expectations(df):
+    """自動でバリデーションルールを推論。"""
+    expectations = []
+    for col in df.columns:
+        expectations.append({"type": "expect_column_to_exist", "column": col})
+        expectations.append({
+            "type": "expect_column_values_to_not_be_null",
+            "column": col,
+            "kwargs": {"mostly": 0.9}})
+
+        if df[col].dtype in [np.float64, np.int64]:
+            q1, q3 = df[col].quantile([0.01, 0.99])
+            iqr = q3 - q1
+            expectations.append({
+                "type": "expect_column_values_to_be_between",
+                "column": col,
+                "kwargs": {"min_value": float(q1 - 3 * iqr),
+                           "max_value": float(q3 + 3 * iqr)}})
+    return expectations
+```
+
+---
+
+## パイプライン統合
+
+```
+[データ取得] → data-profiling → eda-correlation
+                (品質診断)       (探索的解析)
+                     │                ↓
+          missing-data-analysis  anomaly-detection
+            (欠損補完)             (異常検知)
+```
+
+## パイプライン出力
+
+| ファイル | 説明 | 次スキル |
+|---------|------|---------|
+| `profile_report.html` | ydata-profiling レポート | → EDA |
+| `quality_score.json` | データ品質スコア | → 品質管理 |
+| `validation_results.csv` | バリデーション結果 | → データ修正 |

package/src/.github/skills/scientific-ensemble-methods/SKILL.md

@@ -0,0 +1,263 @@
+---
+name: scientific-ensemble-methods
+description: |
+  アンサンブル学習スキル。Stacking/Blending 多段積層・
+  Boosting (XGBoost/LightGBM/CatBoost) 勾配ブースティング・
+  Bagging/Random Subspace・Voting 分類器/回帰器・
+  アンサンブル多様性評価・モデル統合パイプライン。
+---
+
+# Scientific Ensemble Methods
+
+複数モデルの組み合わせによる予測精度向上・安定化を実現する
+アンサンブル学習手法の設計・評価パイプラインを提供する。
+
+## When to Use
+
+- XGBoost/LightGBM/CatBoost で勾配ブースティングを実行するとき
+- Stacking/Blending で多段アンサンブルを構築するとき
+- 複数モデルの Voting/Averaging で安定予測を得るとき
+- アンサンブルの多様性を評価するとき
+- Out-of-Fold 予測でリーク防止を行うとき
+- モデルの寄与度を分析するとき
+
+---
+
+## Quick Start
+
+## 1. 勾配ブースティング比較
+
+```python
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import cross_val_score
+
+
+def compare_boosting(X, y, cv=5, scoring="f1_macro",
+                     task="classification"):
+    """
+    XGBoost / LightGBM / CatBoost 比較。
+
+    Parameters:
+        X: np.ndarray — 特徴量
+        y: np.ndarray — ラベル
+        cv: int — CV 分割数
+        scoring: str — 評価指標
+        task: str — "classification" / "regression"
+    """
+    results = []
+
+    try:
+        from xgboost import XGBClassifier, XGBRegressor
+        model = (XGBClassifier(n_estimators=200, max_depth=6,
+                               learning_rate=0.1, random_state=42,
+                               use_label_encoder=False, eval_metric="logloss")
+                 if task == "classification"
+                 else XGBRegressor(n_estimators=200, max_depth=6,
+                                   learning_rate=0.1, random_state=42))
+        scores = cross_val_score(model, X, y, cv=cv, scoring=scoring)
+        results.append({"model": "XGBoost", "mean": scores.mean(),
+                        "std": scores.std()})
+    except ImportError:
+        pass
+
+    try:
+        from lightgbm import LGBMClassifier, LGBMRegressor
+        model = (LGBMClassifier(n_estimators=200, max_depth=6,
+                                learning_rate=0.1, random_state=42, verbose=-1)
+                 if task == "classification"
+                 else LGBMRegressor(n_estimators=200, max_depth=6,
+                                    learning_rate=0.1, random_state=42, verbose=-1))
+        scores = cross_val_score(model, X, y, cv=cv, scoring=scoring)
+        results.append({"model": "LightGBM", "mean": scores.mean(),
+                        "std": scores.std()})
+    except ImportError:
+        pass
+
+    try:
+        from catboost import CatBoostClassifier, CatBoostRegressor
+        model = (CatBoostClassifier(iterations=200, depth=6,
+                                    learning_rate=0.1, random_seed=42, verbose=0)
+                 if task == "classification"
+                 else CatBoostRegressor(iterations=200, depth=6,
+                                        learning_rate=0.1, random_seed=42, verbose=0))
+        scores = cross_val_score(model, X, y, cv=cv, scoring=scoring)
+        results.append({"model": "CatBoost", "mean": scores.mean(),
+                        "std": scores.std()})
+    except ImportError:
+        pass
+
+    df = pd.DataFrame(results).sort_values("mean", ascending=False)
+    if not df.empty:
+        print(f"Boosting: best = {df.iloc[0]['model']} "
+              f"({scoring} = {df.iloc[0]['mean']:.4f})")
+    return df
+```
+
+## 2. Stacking アンサンブル
+
+```python
+from sklearn.model_selection import StratifiedKFold
+from sklearn.linear_model import LogisticRegression
+from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
+from sklearn.svm import SVC
+
+
+def stacking_ensemble(X_train, y_train, X_test,
+                      base_models=None, meta_model=None,
+                      n_folds=5):
+    """
+    Stacking アンサンブル (Out-of-Fold 予測)。
+
+    Parameters:
+        X_train: np.ndarray — 学習データ
+        y_train: np.ndarray — 学習ラベル
+        X_test: np.ndarray — テストデータ
+        base_models: list | None — ベースモデル
+        meta_model: classifier | None — メタモデル
+        n_folds: int — CV 分割数
+    """
+    if base_models is None:
+        base_models = [
+            ("rf", RandomForestClassifier(n_estimators=200, random_state=42)),
+            ("gbm", GradientBoostingClassifier(n_estimators=200, random_state=42)),
+            ("svm", SVC(probability=True, random_state=42)),
+        ]
+    if meta_model is None:
+        meta_model = LogisticRegression(max_iter=1000, random_state=42)
+
+    kf = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=42)
+    n_classes = len(np.unique(y_train))
+
+    # Out-of-Fold predictions
+    oof_preds = np.zeros((len(y_train), len(base_models) * n_classes))
+    test_preds = np.zeros((len(X_test), len(base_models) * n_classes))
+
+    for i, (name, model) in enumerate(base_models):
+        col_start = i * n_classes
+        col_end = (i + 1) * n_classes
+        test_fold_preds = np.zeros((len(X_test), n_classes, n_folds))
+
+        for fold, (train_idx, val_idx) in enumerate(kf.split(X_train, y_train)):
+            m = model.__class__(**model.get_params()).fit(
+                X_train[train_idx], y_train[train_idx])
+            oof_preds[val_idx, col_start:col_end] = m.predict_proba(
+                X_train[val_idx])
+            test_fold_preds[:, :, fold] = m.predict_proba(X_test)
+
+        test_preds[:, col_start:col_end] = test_fold_preds.mean(axis=2)
+        print(f"  Stacking base: {name} done")
+
+    # Meta-model
+    meta_model.fit(oof_preds, y_train)
+    final_pred = meta_model.predict(test_preds)
+    final_proba = meta_model.predict_proba(test_preds)
+
+    print(f"Stacking: {len(base_models)} base models → meta-model")
+    return final_pred, final_proba, meta_model
+```
+
+## 3. Voting アンサンブル
+
+```python
+from sklearn.ensemble import VotingClassifier, VotingRegressor
+
+
+def voting_ensemble(X, y, models=None, voting="soft",
+                    cv=5, scoring="f1_macro"):
+    """
+    Voting アンサンブル。
+
+    Parameters:
+        X: np.ndarray — 特徴量
+        y: np.ndarray — ラベル
+        models: list | None — (name, model) ペア
+        voting: str — "soft" / "hard"
+        cv: int — CV 分割数
+        scoring: str — 評価指標
+    """
+    if models is None:
+        models = [
+            ("rf", RandomForestClassifier(n_estimators=200, random_state=42)),
+            ("gbm", GradientBoostingClassifier(n_estimators=200, random_state=42)),
+            ("lr", LogisticRegression(max_iter=1000, random_state=42)),
+        ]
+
+    # 個別モデル評価
+    results = []
+    for name, model in models:
+        scores = cross_val_score(model, X, y, cv=cv, scoring=scoring)
+        results.append({"model": name, "mean": scores.mean(), "std": scores.std()})
+
+    # Voting
+    vc = VotingClassifier(estimators=models, voting=voting)
+    scores = cross_val_score(vc, X, y, cv=cv, scoring=scoring)
+    results.append({"model": f"Voting({voting})",
+                    "mean": scores.mean(), "std": scores.std()})
+
+    df = pd.DataFrame(results).sort_values("mean", ascending=False)
+    print(f"Voting ensemble: {scoring} = {scores.mean():.4f} ± {scores.std():.4f}")
+    return df
+```
+
+## 4. アンサンブル多様性評価
+
+```python
+def ensemble_diversity(models, X, y):
+    """
+    アンサンブル多様性 (Q-statistic / Disagreement)。
+
+    Parameters:
+        models: list — 学習済みモデルリスト
+        X: np.ndarray — 評価データ
+        y: np.ndarray — 真ラベル
+    """
+    predictions = np.array([m.predict(X) for m in models])
+    n_models = len(models)
+    correct = (predictions == y).astype(int)
+
+    # 全ペアの Q-statistic
+    q_stats = []
+    disagree_rates = []
+    for i in range(n_models):
+        for j in range(i + 1, n_models):
+            n11 = np.sum((correct[i] == 1) & (correct[j] == 1))
+            n00 = np.sum((correct[i] == 0) & (correct[j] == 0))
+            n10 = np.sum((correct[i] == 1) & (correct[j] == 0))
+            n01 = np.sum((correct[i] == 0) & (correct[j] == 1))
+
+            denom = n11 * n00 - n10 * n01
+            numer = n11 * n00 + n10 * n01
+            q = denom / numer if numer != 0 else 0
+            q_stats.append(q)
+            disagree_rates.append((n10 + n01) / len(y))
+
+    result = {
+        "mean_q_statistic": round(np.mean(q_stats), 4),
+        "mean_disagreement": round(np.mean(disagree_rates), 4),
+        "n_models": n_models,
+    }
+    print(f"Diversity: Q={result['mean_q_statistic']:.3f}, "
+          f"Disagree={result['mean_disagreement']:.3f}")
+    return result
+```
+
+---
+
+## パイプライン統合
+
+```
+automl → ensemble-methods → uncertainty-quantification
+  (モデル選択)  (アンサンブル)     (不確実性定量化)
+       │            │                    ↓
+  feature-importance ┘         explainable-ai
+    (特徴量重要度)               (説明可能 AI)
+```
+
+## パイプライン出力
+
+| ファイル | 説明 | 次スキル |
+|---------|------|---------|
+| `stacking_meta.pkl` | Stacking メタモデル | → 予測 |
+| `boosting_comparison.csv` | ブースティング比較 | → レポート |
+| `ensemble_diversity.json` | 多様性指標 | → モデル改善 |