m8flow 1.0.1 → 1.1.0

package/bundled/backend/Dockerfile

@@ -0,0 +1,41 @@
+# ── M8Flow Backend ────────────────────────────────────────────────────────────
+# Multi-stage build: all heavy ML packages are baked into the image during
+# `docker build`. The result is a warm container that starts in ~2 seconds
+# with zero pip-install delay at runtime.
+
+FROM python:3.11-slim AS base
+
+WORKDIR /app
+
+# ── System dependencies (needed to compile numpy/scipy/xgboost C extensions) ──
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        gcc g++ libgomp1 curl && \
+    rm -rf /var/lib/apt/lists/*
+
+# ── Python dependencies (separate layer — cached unless requirements.txt changes)
+COPY requirements.txt .
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+# ── Pre-import warmup — compiles .pyc bytecode during build so the first
+#    real request doesn't pay any import overhead.
+COPY warmup.py .
+RUN python warmup.py
+
+# ── Application code (last layer — changes here don't bust the dep cache) ────
+COPY . .
+
+# ── Runtime config ────────────────────────────────────────────────────────────
+ENV PYTHONUNBUFFERED=1
+ENV M8FLOW_ENV=docker
+
+EXPOSE 8000
+
+HEALTHCHECK --interval=10s --timeout=5s --start-period=5s --retries=3 \
+    CMD curl -f http://localhost:8000/api/health || exit 1
+
+CMD ["uvicorn", "main:app", \
+     "--host", "0.0.0.0", \
+     "--port", "8000", \
+     "--log-level", "warning", \
+     "--workers", "1"]

package/bundled/backend/add_nodes.py

@@ -0,0 +1,416 @@
+import re
+
+with open("templates.py", "r") as f:
+    content = f.read()
+
+# The string constants
+NEW_CODE = """
+BINARY_ENCODER = '''import pandas as pd
+try:
+    import category_encoders as ce
+except ImportError:
+    pass
+
+def run(data, columns: str = "") -> dict:
+    df = data.copy()
+    cols = [c.strip() for c in columns.split(",") if c.strip()] if columns.strip() else \
+           df.select_dtypes(include=["object", "category"]).columns.tolist()
+           
+    if not cols:
+        return {"data": df, "summary": "No columns to encode"}
+        
+    try:
+        encoder = ce.BinaryEncoder(cols=cols)
+        df = encoder.fit_transform(df)
+        return {"data": df, "summary": f"Binary encoded {len(cols)} columns"}
+    except NameError:
+        return {"data": df, "summary": "category_encoders not installed. Skipping."}
+'''
+
+FREQUENCY_ENCODER = '''import pandas as pd
+
+def run(data, columns: str = "") -> dict:
+    df = data.copy()
+    cols = [c.strip() for c in columns.split(",") if c.strip()] if columns.strip() else \
+           df.select_dtypes(include=["object", "category"]).columns.tolist()
+           
+    if not cols:
+        return {"data": df, "summary": "No columns to encode"}
+        
+    for col in cols:
+        if col in df.columns:
+            freq = df[col].value_counts(normalize=True)
+            df[col] = df[col].map(freq)
+            
+    return {"data": df, "summary": f"Frequency encoded {len(cols)} columns"}
+'''
+
+ORDINAL_ENCODER = '''import pandas as pd
+from sklearn.preprocessing import OrdinalEncoder
+
+def run(data, columns: str = "") -> dict:
+    df = data.copy()
+    cols = [c.strip() for c in columns.split(",") if c.strip()] if columns.strip() else \
+           df.select_dtypes(include=["object", "category"]).columns.tolist()
+           
+    if not cols:
+        return {"data": df, "summary": "No columns to encode"}
+        
+    encoder = OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=-1)
+    df[cols] = encoder.fit_transform(df[cols].astype(str))
+    
+    return {"data": df, "summary": f"Ordinal encoded {len(cols)} columns"}
+'''
+
+VIF_FEATURE_SELECTION = '''import pandas as pd
+import numpy as np
+from statsmodels.stats.outliers_influence import variance_inflation_factor
+
+def run(data, threshold: float = 5.0) -> dict:
+    df = data.copy()
+    num_cols = df.select_dtypes(include=[np.number]).columns.tolist()
+    
+    if len(num_cols) < 2:
+        return {"data": df, "summary": "Not enough numeric columns for VIF"}
+        
+    X = df[num_cols].dropna()
+    dropped = []
+    
+    while True:
+        vif_data = pd.DataFrame()
+        vif_data["feature"] = X.columns
+        vif_data["VIF"] = [variance_inflation_factor(X.values, i) for i in range(len(X.columns))]
+        
+        max_vif = vif_data["VIF"].max()
+        if max_vif > threshold:
+            max_feat = vif_data.sort_values("VIF", ascending=False).iloc[0]["feature"]
+            X = X.drop(columns=[max_feat])
+            dropped.append(max_feat)
+        else:
+            break
+            
+    df = df.drop(columns=dropped)
+    return {"data": df, "summary": f"Dropped {len(dropped)} features due to high VIF: {dropped}"}
+'''
+
+PCA_WHITENING = '''import pandas as pd
+import numpy as np
+from sklearn.decomposition import PCA
+
+def run(data, n_components: int = 0, whiten: bool = True) -> dict:
+    df = data.copy()
+    num_cols = df.select_dtypes(include=[np.number]).columns.tolist()
+    
+    if not num_cols:
+        return {"data": df, "summary": "No numeric columns for PCA"}
+        
+    n = n_components if n_components > 0 else None
+    pca = PCA(n_components=n, whiten=whiten)
+    
+    pca_features = pca.fit_transform(df[num_cols].fillna(0))
+    
+    feature_names = [f"pca_{i}" for i in range(pca_features.shape[1])]
+    pca_df = pd.DataFrame(pca_features, columns=feature_names, index=df.index)
+    
+    df = df.drop(columns=num_cols)
+    df = pd.concat([df, pca_df], axis=1)
+    
+    return {"data": df, "summary": f"Applied PCA whitening, created {len(feature_names)} components"}
+'''
+
+K_MEANS_CLUSTERING_FEATURES = '''import pandas as pd
+import numpy as np
+from sklearn.cluster import KMeans
+
+def run(data, n_clusters: int = 3, random_state: int = 42) -> dict:
+    df = data.copy()
+    num_cols = df.select_dtypes(include=[np.number]).columns.tolist()
+    
+    if not num_cols:
+        return {"data": df, "summary": "No numeric columns for KMeans"}
+        
+    kmeans = KMeans(n_clusters=n_clusters, random_state=random_state, n_init='auto')
+    df['ClusterID'] = kmeans.fit_predict(df[num_cols].fillna(0))
+    
+    return {"data": df, "summary": f"Added ClusterID with {n_clusters} clusters"}
+'''
+
+XGBOOST_NODE = '''import pandas as pd
+import numpy as np
+
+def run(X_train, X_test, y_train, task_type: str = "classifier", n_estimators: int = 100, learning_rate: float = 0.1, max_depth: int = 3) -> dict:
+    if task_type.lower() == "classifier":
+        from xgboost import XGBClassifier
+        model = XGBClassifier(n_estimators=n_estimators, learning_rate=learning_rate, max_depth=max_depth, use_label_encoder=False, eval_metric='logloss')
+    else:
+        from xgboost import XGBRegressor
+        model = XGBRegressor(n_estimators=n_estimators, learning_rate=learning_rate, max_depth=max_depth)
+        
+    model.fit(X_train, y_train)
+    y_pred = model.predict(X_test)
+    
+    return {"model": model, "y_pred": y_pred}
+'''
+
+LIGHTGBM_NODE = '''import pandas as pd
+
+def run(X_train, X_test, y_train, task_type: str = "classifier", n_estimators: int = 100, learning_rate: float = 0.1, max_depth: int = -1) -> dict:
+    if task_type.lower() == "classifier":
+        from lightgbm import LGBMClassifier
+        model = LGBMClassifier(n_estimators=n_estimators, learning_rate=learning_rate, max_depth=max_depth)
+    else:
+        from lightgbm import LGBMRegressor
+        model = LGBMRegressor(n_estimators=n_estimators, learning_rate=learning_rate, max_depth=max_depth)
+        
+    model.fit(X_train, y_train)
+    y_pred = model.predict(X_test)
+    
+    return {"model": model, "y_pred": y_pred}
+'''
+
+ADABOOST_NODE = '''import pandas as pd
+
+def run(X_train, X_test, y_train, task_type: str = "classifier", n_estimators: int = 50, learning_rate: float = 1.0) -> dict:
+    if task_type.lower() == "classifier":
+        from sklearn.ensemble import AdaBoostClassifier
+        model = AdaBoostClassifier(n_estimators=n_estimators, learning_rate=learning_rate)
+    else:
+        from sklearn.ensemble import AdaBoostRegressor
+        model = AdaBoostRegressor(n_estimators=n_estimators, learning_rate=learning_rate)
+        
+    model.fit(X_train, y_train)
+    y_pred = model.predict(X_test)
+    
+    return {"model": model, "y_pred": y_pred}
+'''
+
+VOTING_ENSEMBLE = '''import pandas as pd
+import numpy as np
+
+def run(model1, model2, model3, X_train, X_test, y_train, task_type: str = "classifier", voting: str = "hard") -> dict:
+    estimators = [('m1', model1), ('m2', model2), ('m3', model3)]
+    
+    if task_type.lower() == "classifier":
+        from sklearn.ensemble import VotingClassifier
+        model = VotingClassifier(estimators=estimators, voting=voting)
+    else:
+        from sklearn.ensemble import VotingRegressor
+        model = VotingRegressor(estimators=estimators)
+        
+    model.fit(X_train, y_train)
+    y_pred = model.predict(X_test)
+    
+    return {"model": model, "y_pred": y_pred}
+'''
+
+LAG_FEATURE_GENERATOR = '''import pandas as pd
+from typing import Annotated
+
+def run(data, column: Annotated[str, "column"], lags: int = 3) -> dict:
+    df = data.copy()
+    if column not in df.columns:
+        raise ValueError(f"Column '{column}' not found.")
+        
+    for i in range(1, lags + 1):
+        df[f"{column}_lag_{i}"] = df[column].shift(i)
+        
+    return {"data": df, "summary": f"Created {lags} lag features for {column}"}
+'''
+
+ROLLING_WINDOW_STATS = '''import pandas as pd
+from typing import Annotated
+
+def run(data, column: Annotated[str, "column"], window: int = 7) -> dict:
+    df = data.copy()
+    if column not in df.columns:
+        raise ValueError(f"Column '{column}' not found.")
+        
+    df[f"{column}_roll_mean_{window}"] = df[column].rolling(window=window).mean()
+    df[f"{column}_roll_std_{window}"] = df[column].rolling(window=window).std()
+    
+    return {"data": df, "summary": f"Created rolling window ({window}) stats for {column}"}
+'''
+
+PERMUTATION_IMPORTANCE = '''import numpy as np
+from sklearn.inspection import permutation_importance
+
+def run(model, X_test, y_test, scoring: str = "accuracy", n_repeats: int = 5, random_state: int = 42) -> dict:
+    result = permutation_importance(model, X_test, y_test, scoring=scoring, n_repeats=n_repeats, random_state=random_state)
+    
+    importances = result.importances_mean
+    
+    if hasattr(X_test, 'columns'):
+        names = X_test.columns
+    else:
+        names = [f"Feature {i}" for i in range(len(importances))]
+        
+    indices = np.argsort(importances)[::-1]
+    
+    top_importances = importances[indices].tolist()
+    top_names = [str(names[i]) for i in indices]
+    
+    return {
+        "features": top_names,
+        "importances": top_importances,
+        "summary": "Permutation Importance"
+    }
+'''
+
+LEARNING_CURVE_DATA = '''import numpy as np
+from sklearn.model_selection import learning_curve
+
+def run(model, X_train, y_train, cv_folds: int = 5, scoring: str = "accuracy") -> dict:
+    train_sizes, train_scores, test_scores = learning_curve(
+        model, X_train, y_train, cv=cv_folds, scoring=scoring,
+        train_sizes=np.linspace(0.1, 1.0, 10), random_state=42
+    )
+    
+    train_mean = np.mean(train_scores, axis=1).tolist()
+    test_mean = np.mean(test_scores, axis=1).tolist()
+    sizes = train_sizes.tolist()
+    
+    return {
+        "train_sizes": sizes,
+        "train_scores": train_mean,
+        "val_scores": test_mean,
+        "scoring": scoring,
+        "summary": "Learning Curve Data"
+    }
+'''
+
+LIFT_GAIN_CHARTS = '''import numpy as np
+
+def run(model, X_test, y_test) -> dict:
+    if not hasattr(model, "predict_proba"):
+        raise ValueError("Model does not support predict_proba required for lift/gain charts.")
+        
+    classes = model.classes_
+    if len(classes) != 2:
+        raise ValueError("Lift/Gain charts require binary classification.")
+        
+    y_scores = model.predict_proba(X_test)[:, 1]
+    pos_label = classes[1]
+    
+    y_true_bin = (np.array(y_test) == pos_label).astype(int)
+    
+    indices = np.argsort(y_scores)[::-1]
+    y_true_sorted = y_true_bin[indices]
+    
+    total_positives = y_true_bin.sum()
+    total_samples = len(y_true_bin)
+    
+    cum_positives = np.cumsum(y_true_sorted)
+    
+    gain = cum_positives / max(total_positives, 1)
+    
+    count = np.arange(1, total_samples + 1)
+    lift = (cum_positives / count) / (total_positives / total_samples)
+    
+    deciles = np.linspace(0, 1, 11)[1:]
+    gain_deciles = [gain[min(int(d * total_samples) - 1, total_samples - 1)] for d in deciles]
+    lift_deciles = [lift[min(int(d * total_samples) - 1, total_samples - 1)] for d in deciles]
+    
+    return {
+        "deciles": deciles.tolist(),
+        "gain": gain_deciles,
+        "lift": lift_deciles,
+        "summary": "Lift and Gain Data"
+    }
+'''
+
+TOMEK_LINKS = '''import pandas as pd
+import numpy as np
+from typing import Annotated
+
+def run(data, target_column: Annotated[str, "column"] = "target") -> dict:
+    df = data.copy()
+    if target_column not in df.columns:
+        raise ValueError(f"Target column '{target_column}' not found.")
+        
+    X = df.drop(columns=[target_column])
+    y = df[target_column]
+    
+    try:
+        from imblearn.under_sampling import TomekLinks
+        tl = TomekLinks()
+        X_res, y_res = tl.fit_resample(X, y)
+        df_out = X_res.copy()
+        df_out[target_column] = y_res
+        
+        return {"data": df_out, "summary": f"Tomek Links removed {len(df) - len(df_out)} noisy samples."}
+    except ImportError:
+        return {"data": df, "summary": "imbalanced-learn not installed. Skipping Tomek Links."}
+'''
+
+RANDOM_UNDER_SAMPLER = '''import pandas as pd
+from typing import Annotated
+
+def run(data, target_column: Annotated[str, "column"] = "target", random_state: int = 42) -> dict:
+    df = data.copy()
+    if target_column not in df.columns:
+        raise ValueError(f"Target column '{target_column}' not found.")
+        
+    try:
+        from imblearn.under_sampling import RandomUnderSampler
+        X = df.drop(columns=[target_column])
+        y = df[target_column]
+        rus = RandomUnderSampler(random_state=random_state)
+        X_res, y_res = rus.fit_resample(X, y)
+        df_out = X_res.copy()
+        df_out[target_column] = y_res
+        
+        return {"data": df_out, "summary": f"Undersampled from {len(df)} to {len(df_out)} rows."}
+    except ImportError:
+        min_class_size = df[target_column].value_counts().min()
+        df_out = df.groupby(target_column).sample(n=min_class_size, random_state=random_state)
+        return {"data": df_out, "summary": f"Manual undersampled from {len(df)} to {len(df_out)} rows."}
+'''
+"""
+
+# Find where TEMPLATES = [ starts
+parts = content.split("TEMPLATES: list[dict] = [")
+before_templates = parts[0]
+templates_list = parts[1]
+
+# Insert the new code before TEMPLATES
+new_content = before_templates + NEW_CODE + "\nTEMPLATES: list[dict] = ["
+
+# PREPROCESSING (find # Preprocessing)
+templates_list = templates_list.replace(
+    '    {"id": "datetime_extractor",          "label": "Datetime Extractor",      "category": "Preprocessing", "code": DATETIME_EXTRACTOR},',
+    '''    {"id": "datetime_extractor",          "label": "Datetime Extractor",      "category": "Preprocessing", "code": DATETIME_EXTRACTOR},
+    {"id": "binary_encoder",              "label": "Binary Encoder",          "category": "Preprocessing", "code": BINARY_ENCODER},
+    {"id": "frequency_encoder",           "label": "Frequency Encoder",       "category": "Preprocessing", "code": FREQUENCY_ENCODER},
+    {"id": "ordinal_encoder",             "label": "Ordinal Encoder",         "category": "Preprocessing", "code": ORDINAL_ENCODER},
+    {"id": "vif_feature_selection",       "label": "VIF Feature Selection",   "category": "Preprocessing", "code": VIF_FEATURE_SELECTION},
+    {"id": "pca_whitening",               "label": "PCA Whitening",           "category": "Preprocessing", "code": PCA_WHITENING},
+    {"id": "k_means_clustering_features", "label": "K-Means Features",        "category": "Preprocessing", "code": K_MEANS_CLUSTERING_FEATURES},
+    {"id": "lag_feature_generator",       "label": "Lag Features",            "category": "Preprocessing", "code": LAG_FEATURE_GENERATOR},
+    {"id": "rolling_window_stats",        "label": "Rolling Window Stats",    "category": "Preprocessing", "code": ROLLING_WINDOW_STATS},
+    {"id": "tomek_links",                 "label": "Tomek Links",             "category": "Preprocessing", "code": TOMEK_LINKS},
+    {"id": "random_under_sampler",        "label": "Random Under Sampler",    "category": "Preprocessing", "code": RANDOM_UNDER_SAMPLER},'''
+)
+
+# CLASSIFIERS (find # Classifiers)
+templates_list = templates_list.replace(
+    '    {"id": "knn_classifier",              "label": "KNN Classifier",          "category": "Classifiers",   "code": KNN_CLASSIFIER},',
+    '''    {"id": "knn_classifier",              "label": "KNN Classifier",          "category": "Classifiers",   "code": KNN_CLASSIFIER},
+    {"id": "xgboost_node",                "label": "XGBoost",                 "category": "Classifiers",   "code": XGBOOST_NODE},
+    {"id": "lightgbm_node",               "label": "LightGBM",                "category": "Classifiers",   "code": LIGHTGBM_NODE},
+    {"id": "adaboost_node",               "label": "AdaBoost",                "category": "Classifiers",   "code": ADABOOST_NODE},
+    {"id": "voting_ensemble",             "label": "Voting Ensemble",         "category": "Classifiers",   "code": VOTING_ENSEMBLE},'''
+)
+
+# EVALUATION
+templates_list = templates_list.replace(
+    '    {"id": "auto_ml",                     "label": "AutoML",                  "category": "Evaluation",    "code": AUTO_ML},',
+    '''    {"id": "permutation_importance",      "label": "Permutation Importance",  "category": "Evaluation",    "code": PERMUTATION_IMPORTANCE},
+    {"id": "learning_curve_data",         "label": "Learning Curve",          "category": "Evaluation",    "code": LEARNING_CURVE_DATA},
+    {"id": "lift_gain_charts",            "label": "Lift & Gain Charts",      "category": "Evaluation",    "code": LIFT_GAIN_CHARTS},
+    {"id": "auto_ml",                     "label": "AutoML",                  "category": "Evaluation",    "code": AUTO_ML},'''
+)
+
+with open("templates.py", "w") as f:
+    f.write(new_content + templates_list)
+
+print("success")

package/bundled/backend/api/routes/appstate.py

@@ -0,0 +1,102 @@
+"""
+App-state persistence — saves and loads the full frontend state
+(flows, projects, myFiles) to/from a single JSON file on the server.
+
+This makes the app work consistently across any browser on the same machine,
+because all browsers hit the same localhost backend which reads the same file.
+"""
+import json
+import logging
+import os
+from pathlib import Path
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+def _state_file() -> Path:
+    """Return the path to app_state.json, creating parent dirs if needed."""
+    base = Path(
+        os.environ.get("M8FLOW_PIPELINE_DIR")
+        or (Path.home() / ".m8flow" / "pipelines")
+    )
+    base.mkdir(parents=True, exist_ok=True)
+    return base.parent / "app_state.json"
+
+
+@router.get("")
+def get_state():
+    """Load the full app state from disk. Returns empty defaults if no file yet."""
+    f = _state_file()
+    if f.exists():
+        try:
+            return json.loads(f.read_text(encoding="utf-8"))
+        except Exception as exc:
+            logger.warning("app_state.json unreadable: %s — returning defaults", exc)
+    return {"flows": [], "projects": [], "myFiles": [], "openRouterKey": None}
+
+
+class StatePayload(BaseModel):
+    flows:          list = []
+    projects:       list = []
+    myFiles:        list = []
+    openRouterKey:  str | None = None
+
+
+@router.post("")
+def save_state(payload: StatePayload):
+    """Persist the full app state to disk atomically."""
+    f = _state_file()
+    try:
+        tmp = f.with_suffix(".tmp")
+        tmp.write_text(
+            json.dumps(payload.model_dump(), ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+        tmp.replace(f)
+    except Exception as exc:
+        logger.error("Failed to save app_state: %s", exc)
+        raise HTTPException(status_code=500, detail=f"State save failed: {exc}")
+    return {"ok": True}
+
+
+# ── Dedicated key endpoints (used by settings UI) ─────────────────────────────
+
+class KeyPayload(BaseModel):
+    key: str | None = None
+
+
+@router.get("/api-key")
+def get_api_key():
+    """Return the stored OpenRouter key (masked for security in logs, full for UI)."""
+    f = _state_file()
+    if f.exists():
+        try:
+            data = json.loads(f.read_text(encoding="utf-8"))
+            return {"key": data.get("openRouterKey") or None}
+        except Exception:
+            pass
+    return {"key": None}
+
+
+@router.post("/api-key")
+def save_api_key(payload: KeyPayload):
+    """Save or clear the OpenRouter key server-side."""
+    f = _state_file()
+    data: dict = {"flows": [], "projects": [], "myFiles": [], "openRouterKey": None}
+    if f.exists():
+        try:
+            data = json.loads(f.read_text(encoding="utf-8"))
+        except Exception:
+            pass
+    data["openRouterKey"] = payload.key or None
+    try:
+        tmp = f.with_suffix(".tmp")
+        tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
+        tmp.replace(f)
+    except Exception as exc:
+        raise HTTPException(status_code=500, detail=f"Key save failed: {exc}")
+    return {"ok": True}