@nahisaho/satori 0.23.0 → 0.24.0

package/src/.github/skills/scientific-causal-ml/SKILL.md

@@ -0,0 +1,240 @@
+---
+name: scientific-causal-ml
+description: |
+  因果機械学習スキル。DoWhy 因果モデル・EconML CATE 推定・
+  Double/Debiased ML・Causal Forest・メタラーナー (S/T/X)・
+  異質的処置効果 (HTE)・因果特徴量選択。
+---
+
+# Scientific Causal ML
+
+機械学習ベースの因果推論パイプラインを提供し、
+異質的処置効果 (HTE) の推定と因果特徴量発見を実現する。
+
+## When to Use
+
+- 処置効果が個人/サブグループで異なるとき (HTE 推定)
+- Causal Forest で非パラメトリック因果効果を推定するとき
+- Double ML で高次元データの処置効果を推定するとき
+- メタラーナー (S/T/X-learner) で CATE を推定するとき
+- DoWhy で因果モデルの同定・推定・反論をするとき
+- 因果特徴量選択で重要な効果修飾因子を発見するとき
+
+> **Note**: 統計的因果推論 (PSM/IPW/DID/RDD) は `scientific-causal-inference` を参照。
+
+---
+
+## Quick Start
+
+## 1. DoWhy 因果モデル
+
+```python
+import numpy as np
+import pandas as pd
+
+
+def dowhy_causal_model(df, treatment, outcome, common_causes,
+                       effect_modifiers=None, method="backdoor.linear_regression"):
+    """
+    DoWhy 因果推論パイプライン (同定→推定→反論)。
+
+    Parameters:
+        df: pd.DataFrame — 観測データ
+        treatment: str — 処置変数
+        outcome: str — 結果変数
+        common_causes: list[str] — 共変量
+        effect_modifiers: list[str] | None — 効果修飾因子
+        method: str — 推定手法
+    """
+    import dowhy
+
+    model = dowhy.CausalModel(
+        data=df,
+        treatment=treatment,
+        outcome=outcome,
+        common_causes=common_causes,
+        effect_modifiers=effect_modifiers)
+
+    # 同定
+    estimand = model.identify_effect(proceed_when_unidentifiable=True)
+    print(f"Identified estimand: {estimand.get_frontdoor_variables()}")
+
+    # 推定
+    estimate = model.estimate_effect(
+        estimand, method_name=method)
+    print(f"ATE = {estimate.value:.4f} (95% CI: [{estimate.get_confidence_intervals()[0]:.4f}, "
+          f"{estimate.get_confidence_intervals()[1]:.4f}])")
+
+    # 反論テスト
+    refutations = {}
+    for refuter_name in ["random_common_cause", "placebo_treatment_refuter",
+                         "data_subset_refuter"]:
+        try:
+            refutation = model.refute_estimate(
+                estimand, estimate, method_name=refuter_name)
+            refutations[refuter_name] = {
+                "new_effect": float(refutation.new_effect),
+                "p_value": getattr(refutation, "refutation_result", {}).get("p_value", None)
+            }
+        except Exception:
+            pass
+
+    print(f"Refutation tests: {len(refutations)} passed")
+    return {"model": model, "estimand": estimand,
+            "estimate": estimate, "refutations": refutations}
+```
+
+## 2. EconML Double ML / Causal Forest
+
+```python
+def double_ml_estimate(df, treatment, outcome, features,
+                       n_splits=5, model_type="linear"):
+    """
+    Double/Debiased ML による処置効果推定。
+
+    Parameters:
+        df: pd.DataFrame — データ
+        treatment: str — 処置変数
+        outcome: str — 結果変数
+        features: list[str] — 共変量
+        n_splits: int — クロスフィッティング分割数
+        model_type: str — "linear" / "forest"
+    """
+    from econml.dml import LinearDML, CausalForestDML
+
+    Y = df[outcome].values
+    T = df[treatment].values
+    X = df[features].values
+
+    if model_type == "linear":
+        est = LinearDML(cv=n_splits, random_state=42)
+    else:
+        est = CausalForestDML(
+            n_estimators=200, cv=n_splits, random_state=42)
+
+    est.fit(Y, T, X=X)
+
+    ate = est.ate(X)
+    ate_ci = est.ate_interval(X, alpha=0.05)
+
+    # CATE (個人レベル)
+    cate = est.effect(X)
+    cate_ci = est.effect_interval(X, alpha=0.05)
+
+    result_df = pd.DataFrame(X, columns=features)
+    result_df["cate"] = cate
+    result_df["cate_lower"] = cate_ci[0]
+    result_df["cate_upper"] = cate_ci[1]
+
+    print(f"Double ML ({model_type}): ATE={ate:.4f} "
+          f"[{ate_ci[0]:.4f}, {ate_ci[1]:.4f}]")
+    print(f"  CATE range: [{cate.min():.4f}, {cate.max():.4f}]")
+    return {"ate": ate, "ate_ci": ate_ci,
+            "cate_df": result_df, "model": est}
+
+
+def causal_forest(df, treatment, outcome, features,
+                  n_estimators=500):
+    """
+    Causal Forest — 非パラメトリック HTE 推定。
+
+    Parameters:
+        df: pd.DataFrame — データ
+        treatment: str — 処置変数 (binary)
+        outcome: str — 結果変数
+        features: list[str] — 共変量
+        n_estimators: int — 木の数
+    """
+    from econml.dml import CausalForestDML
+
+    Y = df[outcome].values
+    T = df[treatment].values
+    X = df[features].values
+
+    cf = CausalForestDML(
+        n_estimators=n_estimators, random_state=42,
+        min_samples_leaf=10)
+    cf.fit(Y, T, X=X)
+
+    cate = cf.effect(X)
+    cate_ci = cf.effect_interval(X, alpha=0.05)
+
+    # 特徴量重要度 (因果)
+    importances = cf.feature_importances_
+    feat_imp = pd.DataFrame({
+        "feature": features,
+        "causal_importance": importances
+    }).sort_values("causal_importance", ascending=False)
+
+    print(f"Causal Forest: {n_estimators} trees, "
+          f"CATE median={np.median(cate):.4f}")
+    print(f"  Top causal features: {feat_imp.head(5).to_dict('records')}")
+    return {"cate": cate, "cate_ci": cate_ci,
+            "feature_importance": feat_imp, "model": cf}
+```
+
+## 3. メタラーナー (S/T/X-Learner)
+
+```python
+def meta_learner(df, treatment, outcome, features,
+                 learner_type="t", base_model=None):
+    """
+    メタラーナーによる CATE 推定。
+
+    Parameters:
+        df: pd.DataFrame — データ
+        treatment: str — 処置変数 (binary 0/1)
+        outcome: str — 結果変数
+        features: list[str] — 共変量
+        learner_type: str — "s" / "t" / "x"
+        base_model: BaseEstimator | None — ベースモデル
+    """
+    from econml.metalearners import SLearner, TLearner, XLearner
+    from sklearn.ensemble import GradientBoostingRegressor
+
+    if base_model is None:
+        base_model = GradientBoostingRegressor(
+            n_estimators=200, max_depth=5, random_state=42)
+
+    Y = df[outcome].values
+    T = df[treatment].values
+    X = df[features].values
+
+    learners = {"s": SLearner, "t": TLearner, "x": XLearner}
+    LearnerClass = learners[learner_type]
+
+    if learner_type == "s":
+        est = LearnerClass(overall_model=base_model)
+    else:
+        est = LearnerClass(models=base_model)
+
+    est.fit(Y, T, X=X)
+    cate = est.effect(X)
+
+    result_df = pd.DataFrame(X, columns=features)
+    result_df["cate"] = cate
+
+    print(f"{learner_type.upper()}-Learner: "
+          f"CATE mean={cate.mean():.4f}, std={cate.std():.4f}")
+    return {"cate": cate, "cate_df": result_df, "model": est}
+```
+
+---
+
+## パイプライン統合
+
+```
+causal-inference → causal-ml → feature-importance
+   (統計的因果)    (因果 ML)     (特徴量解釈)
+        │               │              ↓
+  clinical-trial ───────┘      explainable-ai
+   (臨床試験)                   (説明可能 AI)
+```
+
+## パイプライン出力
+
+| ファイル | 説明 | 次スキル |
+|---------|------|---------|
+| `dowhy_causal_model.json` | DoWhy 因果モデル | → reporting |
+| `cate_estimates.csv` | CATE 推定値 | → precision-medicine |
+| `causal_feature_importance.csv` | 因果特徴量重要度 | → explainable-ai |

package/src/.github/skills/scientific-data-profiling/SKILL.md

@@ -0,0 +1,247 @@
+---
+name: scientific-data-profiling
+description: |
+  データプロファイリング・品質スキル。ydata-profiling 自動 EDA ・
+  Great Expectations データバリデーション・データ品質スコア・
+  型推論・相関検出・外れ値フラグ・データカタログ生成。
+---
+
+# Scientific Data Profiling
+
+データセットの包括的プロファイリング・品質評価・
+自動 EDA レポートパイプラインを提供する。
+
+## When to Use
+
+- 新しいデータセットの全体像を素早く把握するとき
+- データ品質スコアを算出して品質基準をチェックするとき
+- ydata-profiling で自動 EDA レポートを生成するとき
+- Great Expectations でデータバリデーションルールを定義するとき
+- データカタログ (辞書) を自動生成するとき
+- 相関・外れ値・欠損を一括診断するとき
+
+---
+
+## Quick Start
+
+## 1. ydata-profiling 自動 EDA
+
+```python
+import numpy as np
+import pandas as pd
+
+
+def auto_profile_report(df, title="Data Profile Report",
+                        minimal=False, output="profile_report.html"):
+    """
+    ydata-profiling 自動 EDA レポート。
+
+    Parameters:
+        df: pd.DataFrame — 入力データ
+        title: str — レポートタイトル
+        minimal: bool — 軽量モード
+        output: str — 出力 HTML パス
+    """
+    from ydata_profiling import ProfileReport
+
+    profile = ProfileReport(
+        df, title=title, minimal=minimal,
+        correlations={"pearson": {"calculate": True},
+                      "spearman": {"calculate": True},
+                      "kendall": {"calculate": True}},
+        missing_diagrams={"bar": True, "matrix": True, "heatmap": True})
+
+    profile.to_file(output)
+
+    # サマリー抽出
+    desc = profile.get_description()
+    summary = {
+        "n_rows": len(df),
+        "n_cols": len(df.columns),
+        "n_numeric": len(df.select_dtypes(include=[np.number]).columns),
+        "n_categorical": len(df.select_dtypes(include=["object", "category"]).columns),
+        "total_missing": int(df.isnull().sum().sum()),
+        "missing_pct": float(df.isnull().sum().sum() / (len(df) * len(df.columns)) * 100),
+        "n_duplicates": int(df.duplicated().sum()),
+    }
+
+    print(f"Profile Report → {output}")
+    print(f"  {summary['n_rows']} rows × {summary['n_cols']} cols, "
+          f"{summary['missing_pct']:.1f}% missing, "
+          f"{summary['n_duplicates']} duplicates")
+    return {"report_path": output, "summary": summary}
+```
+
+## 2. データ品質スコア
+
+```python
+def data_quality_score(df, rules=None):
+    """
+    データ品質スコア算出 (0-100)。
+
+    Parameters:
+        df: pd.DataFrame — 入力データ
+        rules: dict | None — カスタムルール
+    """
+    scores = {}
+
+    # 1. 完全性 (Completeness) — 非欠損率
+    completeness = 1.0 - df.isnull().sum().sum() / (len(df) * len(df.columns))
+    scores["completeness"] = completeness
+
+    # 2. 一意性 (Uniqueness) — 非重複率
+    uniqueness = 1.0 - df.duplicated().sum() / len(df) if len(df) > 0 else 1.0
+    scores["uniqueness"] = uniqueness
+
+    # 3. 一貫性 (Consistency) — 型一貫性
+    type_consistent = 0
+    for col in df.columns:
+        non_null = df[col].dropna()
+        if len(non_null) == 0:
+            type_consistent += 1
+            continue
+        try:
+            inferred = pd.api.types.infer_dtype(non_null, skipna=True)
+            if inferred not in ["mixed", "mixed-integer"]:
+                type_consistent += 1
+        except Exception:
+            pass
+    consistency = type_consistent / len(df.columns) if len(df.columns) > 0 else 1.0
+    scores["consistency"] = consistency
+
+    # 4. 適時性 (Timeliness) — 日付カラムの新しさ
+    date_cols = df.select_dtypes(include=["datetime64"]).columns
+    if len(date_cols) > 0:
+        max_date = df[date_cols[0]].max()
+        freshness = 1.0  # Placeholder
+        scores["timeliness"] = freshness
+    else:
+        scores["timeliness"] = 1.0
+
+    # 5. 妥当性 (Validity) — 数値カラムの有限性
+    numeric_cols = df.select_dtypes(include=[np.number]).columns
+    if len(numeric_cols) > 0:
+        finite_rate = df[numeric_cols].apply(lambda x: np.isfinite(x.dropna()).mean()).mean()
+        scores["validity"] = float(finite_rate)
+    else:
+        scores["validity"] = 1.0
+
+    # 総合スコア
+    weights = {"completeness": 0.3, "uniqueness": 0.2,
+               "consistency": 0.2, "timeliness": 0.1, "validity": 0.2}
+    total_score = sum(scores[k] * weights[k] for k in weights) * 100
+
+    # カスタムルール
+    rule_results = []
+    if rules:
+        for rule_name, rule_fn in rules.items():
+            try:
+                passed = rule_fn(df)
+                rule_results.append({"rule": rule_name, "passed": passed})
+            except Exception as e:
+                rule_results.append({"rule": rule_name, "passed": False,
+                                     "error": str(e)})
+
+    print(f"Data Quality Score: {total_score:.1f}/100")
+    for k, v in scores.items():
+        print(f"  {k}: {v:.3f}")
+
+    return {"total_score": total_score, "dimension_scores": scores,
+            "rule_results": rule_results}
+```
+
+## 3. Great Expectations バリデーション
+
+```python
+def great_expectations_validate(df, expectations=None):
+    """
+    Great Expectations スタイルのデータバリデーション。
+
+    Parameters:
+        df: pd.DataFrame — 入力データ
+        expectations: list[dict] | None — バリデーションルール
+    """
+    if expectations is None:
+        expectations = _auto_generate_expectations(df)
+
+    results = []
+    for exp in expectations:
+        exp_type = exp["type"]
+        col = exp.get("column")
+        kwargs = exp.get("kwargs", {})
+
+        try:
+            if exp_type == "expect_column_to_exist":
+                success = col in df.columns
+            elif exp_type == "expect_column_values_to_not_be_null":
+                max_pct = kwargs.get("mostly", 1.0)
+                non_null_pct = df[col].notnull().mean()
+                success = non_null_pct >= max_pct
+            elif exp_type == "expect_column_values_to_be_between":
+                min_val, max_val = kwargs["min_value"], kwargs["max_value"]
+                vals = df[col].dropna()
+                success = bool((vals >= min_val).all() and (vals <= max_val).all())
+            elif exp_type == "expect_column_values_to_be_unique":
+                success = not df[col].duplicated().any()
+            elif exp_type == "expect_column_values_to_be_in_set":
+                valid_set = set(kwargs["value_set"])
+                success = df[col].dropna().isin(valid_set).all()
+            elif exp_type == "expect_table_row_count_to_be_between":
+                success = kwargs["min_value"] <= len(df) <= kwargs["max_value"]
+            else:
+                success = None
+
+            results.append({"expectation": exp_type, "column": col,
+                            "success": success})
+        except Exception as e:
+            results.append({"expectation": exp_type, "column": col,
+                            "success": False, "error": str(e)})
+
+    results_df = pd.DataFrame(results)
+    n_pass = results_df["success"].sum()
+    n_total = len(results_df)
+    print(f"Validation: {n_pass}/{n_total} expectations passed "
+          f"({n_pass/n_total*100:.0f}%)")
+    return results_df
+
+
+def _auto_generate_expectations(df):
+    """自動でバリデーションルールを推論。"""
+    expectations = []
+    for col in df.columns:
+        expectations.append({"type": "expect_column_to_exist", "column": col})
+        expectations.append({
+            "type": "expect_column_values_to_not_be_null",
+            "column": col,
+            "kwargs": {"mostly": 0.9}})
+
+        if df[col].dtype in [np.float64, np.int64]:
+            q1, q3 = df[col].quantile([0.01, 0.99])
+            iqr = q3 - q1
+            expectations.append({
+                "type": "expect_column_values_to_be_between",
+                "column": col,
+                "kwargs": {"min_value": float(q1 - 3 * iqr),
+                           "max_value": float(q3 + 3 * iqr)}})
+    return expectations
+```
+
+---
+
+## パイプライン統合
+
+```
+[データ取得] → data-profiling → eda-correlation
+                (品質診断)       (探索的解析)
+                     │                ↓
+          missing-data-analysis  anomaly-detection
+            (欠損補完)             (異常検知)
+```
+
+## パイプライン出力
+
+| ファイル | 説明 | 次スキル |
+|---------|------|---------|
+| `profile_report.html` | ydata-profiling レポート | → EDA |
+| `quality_score.json` | データ品質スコア | → 品質管理 |
+| `validation_results.csv` | バリデーション結果 | → データ修正 |