@nahisaho/satori 0.22.0 → 0.23.0

package/src/.github/skills/scientific-missing-data-analysis/SKILL.md

@@ -0,0 +1,312 @@
+---
+name: scientific-missing-data-analysis
+description: |
+  欠損データ解析スキル。欠損パターン診断 (MCAR/MAR/MNAR) ・
+  Little's MCAR テスト・多重代入法 (MICE) ・KNN 補完・
+  MissForest・VAE/GAIN 補完・欠損パターン可視化・Rubin's Rules。
+---
+
+# Scientific Missing Data Analysis
+
+欠損データの診断・補完・感度分析パイプラインを提供し、
+バイアスのない統計推論を実現する。
+
+## When to Use
+
+- データセットの欠損パターンを診断するとき
+- MCAR / MAR / MNAR のメカニズムを判定するとき
+- 多重代入法 (MICE) で欠損値を補完するとき
+- KNN / MissForest / 深層学習ベースの補完をするとき
+- 複数の補完結果を Rubin's Rules で統合するとき
+- 欠損パターンを可視化するとき
+
+---
+
+## Quick Start
+
+## 1. 欠損パターン診断
+
+```python
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+
+
+def diagnose_missing_patterns(df, output_prefix="missing"):
+    """
+    欠損パターン診断 — MCAR/MAR/MNAR 判定支援。
+
+    Parameters:
+        df: pd.DataFrame — 入力データ
+        output_prefix: str — 出力ファイル接頭辞
+    """
+    n_rows, n_cols = df.shape
+    missing_counts = df.isnull().sum()
+    missing_pct = (missing_counts / n_rows * 100).round(2)
+
+    summary = pd.DataFrame({
+        "column": df.columns,
+        "n_missing": missing_counts.values,
+        "pct_missing": missing_pct.values,
+        "dtype": df.dtypes.values
+    }).sort_values("pct_missing", ascending=False)
+
+    # 欠損パターン行列 (msno 風)
+    fig, axes = plt.subplots(2, 2, figsize=(16, 12))
+
+    # (1) 欠損マトリックス
+    ax = axes[0, 0]
+    missing_matrix = df.isnull().astype(int)
+    ax.imshow(missing_matrix.values[:200], aspect="auto", cmap="Greys",
+              interpolation="none")
+    ax.set_xlabel("Features")
+    ax.set_ylabel("Samples")
+    ax.set_title("Missing Pattern Matrix (first 200 rows)")
+
+    # (2) 欠損率バー
+    ax = axes[0, 1]
+    cols_with_missing = summary[summary["pct_missing"] > 0]
+    ax.barh(cols_with_missing["column"], cols_with_missing["pct_missing"])
+    ax.set_xlabel("Missing %")
+    ax.set_title("Missing Rate per Column")
+
+    # (3) 欠損相関ヒートマップ
+    ax = axes[1, 0]
+    miss_corr = df.isnull().corr()
+    sns.heatmap(miss_corr, ax=ax, cmap="RdBu_r", center=0,
+                square=True, cbar_kws={"shrink": 0.8})
+    ax.set_title("Missing Correlation")
+
+    # (4) 欠損パターン上位
+    ax = axes[1, 1]
+    patterns = df.isnull().apply(lambda x: tuple(x), axis=1)
+    pattern_counts = patterns.value_counts().head(10)
+    ax.barh(range(len(pattern_counts)),
+            pattern_counts.values)
+    ax.set_yticks(range(len(pattern_counts)))
+    ax.set_yticklabels([str(p)[:40] for p in pattern_counts.index],
+                       fontsize=7)
+    ax.set_xlabel("Count")
+    ax.set_title("Top 10 Missing Patterns")
+
+    plt.tight_layout()
+    path = f"{output_prefix}_diagnosis.png"
+    plt.savefig(path, dpi=150, bbox_inches="tight")
+    plt.close()
+
+    print(f"Missing Diagnosis: {n_cols} cols, "
+          f"{missing_counts.sum()} total missing ({(missing_counts.sum()/(n_rows*n_cols)*100):.1f}%)")
+    return {"summary": summary, "fig": path}
+```
+
+## 2. Little's MCAR テスト
+
+```python
+def littles_mcar_test(df):
+    """
+    Little's MCAR テスト — 完全ランダム欠損の検定。
+
+    Parameters:
+        df: pd.DataFrame — 数値データのみ
+    Returns:
+        dict — chi2 統計量, p値, 判定
+    """
+    from scipy import stats
+
+    numeric_df = df.select_dtypes(include=[np.number])
+    n_rows, n_cols = numeric_df.shape
+
+    # 欠損パターンごとにグルーピング
+    patterns = numeric_df.isnull().apply(tuple, axis=1)
+    unique_patterns = patterns.unique()
+
+    # 全体平均と全体共分散
+    global_mean = numeric_df.mean()
+    global_cov = numeric_df.cov()
+
+    chi2_stat = 0.0
+    df_stat = 0
+
+    for pattern in unique_patterns:
+        mask = patterns == pattern
+        sub_df = numeric_df[mask]
+        n_j = len(sub_df)
+        if n_j < 2:
+            continue
+
+        # このパターンで観測されているカラム
+        obs_cols = [i for i, m in enumerate(pattern) if not m]
+        if len(obs_cols) == 0:
+            continue
+
+        obs_mean = sub_df.iloc[:, obs_cols].mean().values
+        exp_mean = global_mean.iloc[obs_cols].values
+        diff = obs_mean - exp_mean
+
+        obs_cov = global_cov.iloc[obs_cols, obs_cols].values
+        try:
+            cov_inv = np.linalg.pinv(obs_cov / n_j)
+        except np.linalg.LinAlgError:
+            continue
+
+        chi2_stat += diff @ cov_inv @ diff
+        df_stat += len(obs_cols)
+
+    df_stat -= n_cols  # 自由度補正
+
+    if df_stat <= 0:
+        return {"chi2": np.nan, "p_value": np.nan,
+                "conclusion": "判定不能 (自由度不足)"}
+
+    p_value = 1 - stats.chi2.cdf(chi2_stat, df_stat)
+    conclusion = "MCAR (p > 0.05)" if p_value > 0.05 else "Not MCAR (p ≤ 0.05)"
+
+    print(f"Little's MCAR test: χ²={chi2_stat:.2f}, df={df_stat}, "
+          f"p={p_value:.4f} → {conclusion}")
+    return {"chi2": chi2_stat, "df": df_stat,
+            "p_value": p_value, "conclusion": conclusion}
+```
+
+## 3. 多重代入法 (MICE)
+
+```python
+def mice_imputation(df, n_imputations=5, max_iter=10, random_state=42):
+    """
+    MICE (Multiple Imputation by Chained Equations)。
+
+    Parameters:
+        df: pd.DataFrame — 欠損を含むデータ
+        n_imputations: int — 代入データセット数
+        max_iter: int — 反復回数
+        random_state: int — 乱数シード
+    """
+    from sklearn.experimental import enable_iterative_imputer  # noqa
+    from sklearn.impute import IterativeImputer
+
+    numeric_cols = df.select_dtypes(include=[np.number]).columns
+    cat_cols = df.select_dtypes(exclude=[np.number]).columns
+
+    imputed_datasets = []
+
+    for i in range(n_imputations):
+        imputer = IterativeImputer(
+            max_iter=max_iter,
+            random_state=random_state + i,
+            sample_posterior=True)
+
+        imputed_numeric = pd.DataFrame(
+            imputer.fit_transform(df[numeric_cols]),
+            columns=numeric_cols, index=df.index)
+
+        imputed_df = imputed_numeric.copy()
+        for col in cat_cols:
+            imputed_df[col] = df[col].fillna(df[col].mode().iloc[0]
+                                              if not df[col].mode().empty else "UNKNOWN")
+
+        imputed_datasets.append(imputed_df)
+
+    print(f"MICE: {n_imputations} datasets × {max_iter} iterations, "
+          f"{len(numeric_cols)} numeric cols")
+    return imputed_datasets
+
+
+def rubins_rules(estimates, variances):
+    """
+    Rubin's Rules — 多重代入結果の統合。
+
+    Parameters:
+        estimates: list[float] — 各代入データセットからの推定値
+        variances: list[float] — 各代入データセットからの分散
+    """
+    m = len(estimates)
+    Q_bar = np.mean(estimates)
+    U_bar = np.mean(variances)  # Within-imputation variance
+    B = np.var(estimates, ddof=1)  # Between-imputation variance
+    T = U_bar + (1 + 1 / m) * B  # Total variance
+
+    # 自由度 (Barnard-Rubin)
+    r = (1 + 1 / m) * B / U_bar if U_bar > 0 else np.inf
+    df_old = (m - 1) * (1 + 1 / r) ** 2 if r > 0 else np.inf
+
+    print(f"Rubin's Rules: Q̄={Q_bar:.4f}, T={T:.4f}, "
+          f"within={U_bar:.4f}, between={B:.4f}")
+    return {"pooled_estimate": Q_bar, "total_variance": T,
+            "within_variance": U_bar, "between_variance": B,
+            "df": df_old}
+```
+
+## 4. KNN / MissForest 補完
+
+```python
+def knn_imputation(df, n_neighbors=5):
+    """
+    KNN 欠損値補完。
+
+    Parameters:
+        df: pd.DataFrame — 欠損を含むデータ
+        n_neighbors: int — 近傍数
+    """
+    from sklearn.impute import KNNImputer
+
+    numeric_cols = df.select_dtypes(include=[np.number]).columns
+    imputer = KNNImputer(n_neighbors=n_neighbors)
+    imputed = pd.DataFrame(
+        imputer.fit_transform(df[numeric_cols]),
+        columns=numeric_cols, index=df.index)
+
+    n_imputed = df[numeric_cols].isnull().sum().sum()
+    print(f"KNN Imputation (k={n_neighbors}): {n_imputed} values imputed")
+    return imputed
+
+
+def missforest_imputation(df, n_estimators=100, max_iter=10):
+    """
+    MissForest (Random Forest ベースの反復補完)。
+
+    Parameters:
+        df: pd.DataFrame — 欠損を含むデータ
+        n_estimators: int — Random Forest の木の数
+        max_iter: int — 反復回数
+    """
+    from sklearn.experimental import enable_iterative_imputer  # noqa
+    from sklearn.impute import IterativeImputer
+    from sklearn.ensemble import RandomForestRegressor
+
+    numeric_cols = df.select_dtypes(include=[np.number]).columns
+
+    imputer = IterativeImputer(
+        estimator=RandomForestRegressor(n_estimators=n_estimators,
+                                        random_state=42, n_jobs=-1),
+        max_iter=max_iter, random_state=42)
+
+    imputed = pd.DataFrame(
+        imputer.fit_transform(df[numeric_cols]),
+        columns=numeric_cols, index=df.index)
+
+    n_imputed = df[numeric_cols].isnull().sum().sum()
+    print(f"MissForest (n_trees={n_estimators}, iter={max_iter}): "
+          f"{n_imputed} values imputed")
+    return imputed
+```
+
+---
+
+## パイプライン統合
+
+```
+eda-correlation → missing-data-analysis → ml-classification
+  (探索的解析)      (欠損診断・補完)       (モデリング)
+       │                  │                     ↓
+ statistical-testing ────┘          advanced-visualization
+   (統計検定)                         (結果可視化)
+```
+
+## パイプライン出力
+
+| ファイル | 説明 | 次スキル |
+|---------|------|---------|
+| `missing_diagnosis.png` | 欠損パターン可視化 | → reporting |
+| `mcar_test_result.json` | Little's MCAR テスト | → 補完戦略選択 |
+| `imputed_datasets/` | MICE 多重代入データ | → ml-classification |
+| `imputation_comparison.csv` | 補完手法比較 | → 最終選択 |

package/src/.github/skills/scientific-transfer-learning/SKILL.md

@@ -0,0 +1,298 @@
+---
+name: scientific-transfer-learning
+description: |
+  転移学習・ドメイン適応スキル。事前学習モデルファインチューニング・
+  Few-shot / Zero-shot 学習・ドメイン適応 (DA)・
+  知識蒸留・マルチタスク学習・科学ドメイン特化モデル転移。
+---
+
+# Scientific Transfer Learning
+
+事前学習モデルの科学データへの転移・ドメイン適応・
+Few-shot 学習パイプラインを提供する。
+
+## When to Use
+
+- 事前学習済みモデル (ImageNet/BERT) をファインチューニングするとき
+- 小規模科学データセットで高精度を実現したいとき
+- ドメイン適応で異なるデータ分布間のギャップを埋めるとき
+- Few-shot 学習で数例から分類するとき
+- 知識蒸留で大規模モデルを軽量化するとき
+- マルチタスク学習で複数タスクを共同学習するとき
+
+---
+
+## Quick Start
+
+## 1. Vision モデルファインチューニング
+
+```python
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader
+import numpy as np
+
+
+def finetune_vision_model(train_loader, val_loader,
+                          model_name="resnet50",
+                          num_classes=10, epochs=20,
+                          lr=1e-4, freeze_backbone=True):
+    """
+    Vision モデルファインチューニング。
+
+    Parameters:
+        train_loader: DataLoader — 学習データ
+        val_loader: DataLoader — 検証データ
+        model_name: str — "resnet50" / "vit_b_16" / "efficientnet_b0"
+        num_classes: int — クラス数
+        epochs: int — エポック数
+        lr: float — 学習率
+        freeze_backbone: bool — バックボーン凍結
+    """
+    import torchvision.models as models
+
+    # モデルロード
+    model_fn = getattr(models, model_name)
+    weights_name = model_name.replace("_", "").title() + "_Weights"
+    try:
+        weights = getattr(models, weights_name).DEFAULT
+    except AttributeError:
+        weights = "DEFAULT"
+    model = model_fn(weights=weights)
+
+    # 最終層置換
+    if hasattr(model, "fc"):
+        in_features = model.fc.in_features
+        model.fc = nn.Linear(in_features, num_classes)
+    elif hasattr(model, "classifier"):
+        if isinstance(model.classifier, nn.Sequential):
+            in_features = model.classifier[-1].in_features
+            model.classifier[-1] = nn.Linear(in_features, num_classes)
+        else:
+            in_features = model.classifier.in_features
+            model.classifier = nn.Linear(in_features, num_classes)
+    elif hasattr(model, "heads"):
+        in_features = model.heads.head.in_features
+        model.heads.head = nn.Linear(in_features, num_classes)
+
+    # バックボーン凍結
+    if freeze_backbone:
+        for name, param in model.named_parameters():
+            if "fc" not in name and "classifier" not in name and "heads" not in name:
+                param.requires_grad = False
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    model = model.to(device)
+
+    optimizer = torch.optim.AdamW(
+        filter(lambda p: p.requires_grad, model.parameters()), lr=lr)
+    criterion = nn.CrossEntropyLoss()
+    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, epochs)
+
+    best_acc = 0.0
+    history = []
+
+    for epoch in range(epochs):
+        model.train()
+        train_loss = 0.0
+        for X_batch, y_batch in train_loader:
+            X_batch, y_batch = X_batch.to(device), y_batch.to(device)
+            optimizer.zero_grad()
+            outputs = model(X_batch)
+            loss = criterion(outputs, y_batch)
+            loss.backward()
+            optimizer.step()
+            train_loss += loss.item()
+        scheduler.step()
+
+        # Validation
+        model.eval()
+        correct = total = 0
+        with torch.no_grad():
+            for X_batch, y_batch in val_loader:
+                X_batch, y_batch = X_batch.to(device), y_batch.to(device)
+                outputs = model(X_batch)
+                _, predicted = outputs.max(1)
+                total += y_batch.size(0)
+                correct += predicted.eq(y_batch).sum().item()
+
+        val_acc = correct / total
+        history.append({"epoch": epoch, "train_loss": train_loss / len(train_loader),
+                        "val_acc": val_acc})
+        if val_acc > best_acc:
+            best_acc = val_acc
+
+    print(f"Finetune {model_name}: best val acc = {best_acc:.4f}")
+    return model, history
+```
+
+## 2. NLP モデルファインチューニング
+
+```python
+def finetune_text_classifier(train_texts, train_labels,
+                             val_texts, val_labels,
+                             model_name="dmis-lab/biobert-base-cased-v1.2",
+                             num_labels=2, epochs=5, lr=2e-5):
+    """
+    BERT/BioBERT テキスト分類ファインチューニング。
+
+    Parameters:
+        train_texts: list[str] — 学習テキスト
+        train_labels: list[int] — 学習ラベル
+        val_texts: list[str] — 検証テキスト
+        val_labels: list[int] — 検証ラベル
+        model_name: str — HuggingFace モデル名
+        num_labels: int — ラベル数
+        epochs: int — エポック数
+        lr: float — 学習率
+    """
+    from transformers import (
+        AutoTokenizer, AutoModelForSequenceClassification,
+        TrainingArguments, Trainer)
+    from datasets import Dataset
+
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModelForSequenceClassification.from_pretrained(
+        model_name, num_labels=num_labels)
+
+    def tokenize(examples):
+        return tokenizer(examples["text"], truncation=True,
+                         padding="max_length", max_length=512)
+
+    train_ds = Dataset.from_dict({"text": train_texts, "label": train_labels})
+    val_ds = Dataset.from_dict({"text": val_texts, "label": val_labels})
+    train_ds = train_ds.map(tokenize, batched=True)
+    val_ds = val_ds.map(tokenize, batched=True)
+
+    args = TrainingArguments(
+        output_dir="./ft_output", num_train_epochs=epochs,
+        per_device_train_batch_size=16, learning_rate=lr,
+        evaluation_strategy="epoch", save_strategy="epoch",
+        load_best_model_at_end=True, metric_for_best_model="accuracy")
+
+    def compute_metrics(eval_pred):
+        preds = np.argmax(eval_pred.predictions, axis=-1)
+        acc = (preds == eval_pred.label_ids).mean()
+        return {"accuracy": acc}
+
+    trainer = Trainer(model=model, args=args, train_dataset=train_ds,
+                      eval_dataset=val_ds, compute_metrics=compute_metrics)
+    trainer.train()
+
+    metrics = trainer.evaluate()
+    print(f"Finetune {model_name}: val acc = {metrics['eval_accuracy']:.4f}")
+    return model, tokenizer, metrics
+```
+
+## 3. Few-shot 学習
+
+```python
+def prototypical_network(support_X, support_y, query_X,
+                         feature_extractor=None):
+    """
+    Prototypical Network — Few-shot 分類。
+
+    Parameters:
+        support_X: np.ndarray — サポートセット特徴量
+        support_y: np.ndarray — サポートラベル
+        query_X: np.ndarray — クエリセット特徴量
+        feature_extractor: callable | None — 特徴量抽出器
+    """
+    if feature_extractor is not None:
+        support_emb = feature_extractor(support_X)
+        query_emb = feature_extractor(query_X)
+    else:
+        support_emb = support_X
+        query_emb = query_X
+
+    classes = np.unique(support_y)
+    prototypes = np.array([
+        support_emb[support_y == c].mean(axis=0) for c in classes])
+
+    # ユークリッド距離
+    dists = np.array([
+        np.linalg.norm(query_emb - p, axis=1) for p in prototypes]).T
+
+    predictions = classes[np.argmin(dists, axis=1)]
+    confidences = np.exp(-dists.min(axis=1))
+
+    print(f"Few-shot: {len(classes)} classes, "
+          f"{len(support_y)} support → {len(query_X)} query")
+    return predictions, confidences
+```
+
+## 4. 知識蒸留
+
+```python
+def knowledge_distillation(teacher, student, train_loader,
+                           epochs=20, temperature=4.0, alpha=0.7,
+                           lr=1e-3):
+    """
+    知識蒸留 (Teacher → Student)。
+
+    Parameters:
+        teacher: nn.Module — 教師モデル (frozen)
+        student: nn.Module — 生徒モデル
+        train_loader: DataLoader — 学習データ
+        epochs: int — エポック数
+        temperature: float — 蒸留温度
+        alpha: float — soft loss の重み
+        lr: float — 学習率
+    """
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    teacher = teacher.to(device).eval()
+    student = student.to(device)
+
+    optimizer = torch.optim.AdamW(student.parameters(), lr=lr)
+    ce_loss = nn.CrossEntropyLoss()
+    kl_loss = nn.KLDivLoss(reduction="batchmean")
+
+    for epoch in range(epochs):
+        student.train()
+        total_loss = 0.0
+        for X_batch, y_batch in train_loader:
+            X_batch, y_batch = X_batch.to(device), y_batch.to(device)
+
+            with torch.no_grad():
+                teacher_logits = teacher(X_batch)
+
+            student_logits = student(X_batch)
+
+            soft_loss = kl_loss(
+                nn.functional.log_softmax(student_logits / temperature, dim=1),
+                nn.functional.softmax(teacher_logits / temperature, dim=1)
+            ) * (temperature ** 2)
+
+            hard_loss = ce_loss(student_logits, y_batch)
+            loss = alpha * soft_loss + (1 - alpha) * hard_loss
+
+            optimizer.zero_grad()
+            loss.backward()
+            optimizer.step()
+            total_loss += loss.item()
+
+        print(f"  Epoch {epoch}: loss = {total_loss / len(train_loader):.4f}")
+
+    print(f"Distillation: T={temperature}, α={alpha}, {epochs} epochs")
+    return student
+```
+
+---
+
+## パイプライン統合
+
+```
+deep-learning → transfer-learning → active-learning
+  (モデル設計)    (転移・適応)       (効率的ラベル付け)
+       │               │                 ↓
+  healthcare-ai ───────┘       ensemble-methods
+    (臨床 AI)                    (アンサンブル)
+```
+
+## パイプライン出力
+
+| ファイル | 説明 | 次スキル |
+|---------|------|---------|
+| `ft_model.pt` | ファインチューニング済みモデル | → 推論 |
+| `ft_history.csv` | 学習履歴 | → visualization |
+| `few_shot_predictions.csv` | Few-shot 予測 | → 評価 |