m8flow 1.0.2 → 1.1.0

package/bundled/backend/api/routes/flows.py

@@ -13,12 +13,14 @@ router = APIRouter()
 class GenerateRequest(BaseModel):
     prompt: str
     context: str | None = None
+    custom_components: list[dict] | None = None
 
 
 class UpdateRequest(BaseModel):
     prompt: str
     current_flow: FlowSchema
     context: str | None = None
+    custom_components: list[dict] | None = None
 
 
 class AskRequest(BaseModel):
@@ -26,6 +28,7 @@ class AskRequest(BaseModel):
     prompt: str
     current_flow: FlowSchema | None = None
     context: str | None = None
+    custom_components: list[dict] | None = None
 
 
 def _resolve_template_id(node: dict) -> str:
@@ -56,7 +59,7 @@ def _to_canvas_node(node: dict) -> dict:
         code = ""
         label = node_data.get("label") or template_id
 
-    schema = parse_node_code(code).model_dump() if code.strip() else None
+    schema = parse_node_code(code).model_dump()
 
     # Merge schema defaults with explicit values from LLM
     defaults: dict = {}
@@ -94,7 +97,7 @@ async def generate_flow(http_request: Request, req: GenerateRequest):
     if not req.prompt.strip():
         raise HTTPException(status_code=422, detail="Prompt cannot be empty")
     try:
-        flow = await llm_service.generate_flow(req.prompt, context=req.context)
+        flow = await llm_service.generate_flow(req.prompt, context=req.context, custom_components=req.custom_components)
     except ValueError as exc:
         raise HTTPException(status_code=422, detail=str(exc))
     except RuntimeError as exc:
@@ -111,7 +114,7 @@ async def update_flow(http_request: Request, req: UpdateRequest):
     if not req.prompt.strip():
         raise HTTPException(status_code=422, detail="Prompt cannot be empty")
     try:
-        flow = await llm_service.update_flow(req.prompt, req.current_flow, context=req.context)
+        flow = await llm_service.update_flow(req.prompt, req.current_flow, context=req.context, custom_components=req.custom_components)
     except ValueError as exc:
         raise HTTPException(status_code=422, detail=str(exc))
     except RuntimeError as exc:
@@ -129,19 +132,39 @@ async def refine_flow(http_request: Request, req: UpdateRequest):
     if not req.prompt.strip():
         raise HTTPException(status_code=422, detail="Prompt cannot be empty")
     try:
-        patch = await llm_service.refine_flow(req.prompt, req.current_flow, context=req.context)
+        patch = await llm_service.refine_flow(req.prompt, req.current_flow, context=req.context, custom_components=req.custom_components)
     except ValueError as exc:
         raise HTTPException(status_code=422, detail=str(exc))
     except RuntimeError as exc:
         raise HTTPException(status_code=500, detail=str(exc))
 
-    # Expand any "add" nodes through _to_canvas_node so they arrive fully formed.
+    # Expand any "add" or type-swap "update" nodes through _to_canvas_node so they
+    # arrive fully formed (correct templateId, code, schema, label).
     enriched_nodes: list[dict] = []
     for nc in patch.node_changes:
         d = nc.model_dump()
         if nc.action == "add":
             raw_node = {"id": nc.id, "type": nc.type or "", "position": nc.position or {"x": 0, "y": 200}, "data": nc.data or {}}
             d["node"] = _to_canvas_node(raw_node)
+        elif nc.action == "update":
+            # Template swap: LLM might put the new type at the root (nc.type) or inside data
+            new_type = nc.type
+            if not new_type and isinstance(nc.data, dict):
+                new_type = nc.data.get("type")
+
+            if new_type:
+                existing = next(
+                    (n for n in req.current_flow.nodes if n.get("id") == nc.id), {}
+                )
+                raw_node = {
+                    "id":       nc.id,
+                    "type":     new_type,
+                    "position": existing.get("position", {"x": 0, "y": 200}),
+                    "data":     {"values": (nc.data or {}).get("values", {})},
+                }
+                d["node"] = _to_canvas_node(raw_node)
+                # Ensure the new type is passed back to the frontend inside node.data so the store applyPatch picks it up correctly if nc.node is missing for some reason
+                d["type"] = new_type
         enriched_nodes.append(d)
 
     return {
@@ -170,6 +193,7 @@ async def ask_flow(http_request: Request, req: AskRequest):
             prompt=req.prompt,
             current_flow=req.current_flow,
             context=req.context,
+            custom_components=req.custom_components,
         )
     except ValueError as exc:
         raise HTTPException(status_code=422, detail=str(exc))
@@ -202,6 +226,26 @@ async def ask_flow(http_request: Request, req: AskRequest):
                 "data":     nc.data or {},
             }
             d["node"] = _to_canvas_node(raw_node)
+        elif nc.action == "update":
+            # Template swap: LLM might put the new type at the root (nc.type) or inside data
+            new_type = nc.type
+            if not new_type and isinstance(nc.data, dict):
+                new_type = nc.data.get("type")
+
+            if new_type:
+                existing = next(
+                    (n for n in (req.current_flow.nodes if req.current_flow else [])
+                     if n.get("id") == nc.id),
+                    {},
+                )
+                raw_node = {
+                    "id":       nc.id,
+                    "type":     new_type,
+                    "position": existing.get("position", {"x": 0, "y": 200}),
+                    "data":     {"values": (nc.data or {}).get("values", {})},
+                }
+                d["node"] = _to_canvas_node(raw_node)
+                d["type"] = new_type
         enriched_nodes.append(d)
 
     return {
@@ -382,6 +426,21 @@ async def explain_flow_route(http_request: Request, flow: FlowSchema):
         raise HTTPException(status_code=500, detail=f"{type(exc).__name__}: {exc}")
 
 
+class ChatExplanationRequest(BaseModel):
+    question: str
+    explanation: str
+    current_flow: FlowSchema
+
+@router.post("/explain_chat")
+async def explain_chat_route(http_request: Request, req: ChatExplanationRequest):
+    _inject_api_key(http_request)
+    try:
+        response = await llm_service.chat_explanation(req.question, req.explanation, req.current_flow)
+        return {"response": response}
+    except Exception as exc:
+        raise HTTPException(status_code=500, detail=f"{type(exc).__name__}: {exc}")
+
+
 @router.post("/")
 def create_flow(flow: FlowSchema):
     flow_id = flow_service.create_flow(flow)

package/bundled/backend/api/routes/nodes.py

@@ -1,4 +1,4 @@
-from fastapi import APIRouter, UploadFile, File
+from fastapi import APIRouter, HTTPException, Request, UploadFile, File
 from pydantic import BaseModel
 import os
 import shutil
@@ -68,6 +68,30 @@ def download_file(path: str):
     return FileResponse(path, filename=os.path.basename(path))
 
 
+class GenerateCodeRequest(BaseModel):
+    description: str
+
+
+@router.post("/generate-code")
+async def generate_node_code_route(http_request: Request, req: GenerateCodeRequest):
+    """Generate M8Flow-compatible Python node code from a plain-English description."""
+    from services import llm_service
+
+    # Honour the per-request OpenRouter key set by the frontend
+    key = http_request.headers.get("X-OpenRouter-Key")
+    llm_service._request_api_key.set(key or None)
+
+    if not req.description.strip():
+        raise HTTPException(status_code=422, detail="Description cannot be empty")
+    try:
+        code = await llm_service.generate_node_code(req.description)
+        return {"code": code}
+    except RuntimeError as exc:
+        raise HTTPException(status_code=503, detail=str(exc))
+    except Exception as exc:
+        raise HTTPException(status_code=500, detail=f"{type(exc).__name__}: {exc}")
+
+
 @router.get("/templates")
 def list_templates():
     """Return every prebuilt template with its pre-parsed schema attached."""

package/bundled/backend/core/code_validator.py

@@ -40,6 +40,8 @@ ALLOWED_IMPORTS = frozenset({
     # Scientific computing
     "numpy", "pandas", "scipy", "sklearn", "xgboost", "lightgbm",
     "statsmodels", "imblearn",
+    # Plotting
+    "matplotlib", "seaborn", "plotly",
     # Standard safe libs
     "math", "statistics", "itertools", "functools", "collections",
     "json", "re", "datetime", "typing",

package/bundled/backend/core/executor.py

@@ -24,15 +24,31 @@ def _serialize_value(val: Any) -> Any:
     """Recursively convert Python/pandas/numpy objects to JSON-safe dicts."""
     if isinstance(val, pd.DataFrame):
         null_counts = val.isnull().sum().to_dict()
-        return {
+        result: dict = {
             "_type": "DataFrame",
             "shape": list(val.shape),
             "columns": list(val.columns),
             "dtypes": {c: str(t) for c, t in val.dtypes.items()},
-            "head": val.head(3).fillna("").astype(str).to_dict(orient="records"),
+            "head": val.head(20).fillna("").astype(str).to_dict(orient="records"),
             "null_counts": {k: int(v) for k, v in null_counts.items() if v > 0},
             "null_total": int(val.isnull().sum().sum()),
         }
+
+        # ── Auto-scatter for 2-column numeric DataFrames ───────────────────
+        # Lets ANY custom visualisation node (t-SNE, UMAP, PCA 2D, …) get a
+        # viz button automatically — without requiring custom frontend code.
+        num_cols = val.select_dtypes(include=[np.number]).columns.tolist()
+        if len(num_cols) >= 2:
+            # Sample up to 2 000 points so the chart stays responsive
+            sample = val[num_cols[:2]].dropna()
+            if len(sample) > 2_000:
+                sample = sample.sample(2_000, random_state=42)
+            result["x"]       = [round(float(v), 5) for v in sample[num_cols[0]]]
+            result["y"]       = [round(float(v), 5) for v in sample[num_cols[1]]]
+            result["x_label"] = num_cols[0]
+            result["y_label"] = num_cols[1]
+
+        return result
     if isinstance(val, pd.Series):
         return {
             "_type": "Series",
@@ -77,7 +93,7 @@ def _build_preview(template_id: str, raw_outputs: dict[str, Any]) -> dict:
                 "columns": list(val.columns),
                 "null_total": int(val.isnull().sum().sum()),
                 "null_counts": {c: int(n) for c, n in val.isnull().sum().items() if n > 0},
-                "head": val.head(3).fillna("").astype(str).to_dict(orient="records"),
+                "head": val.head(20).fillna("").astype(str).to_dict(orient="records"),
             }
             # Data-cleaning: capture before/after null comparison
             if "df_before" in raw_outputs and isinstance(raw_outputs["df_before"], pd.DataFrame):

package/bundled/backend/main.py

@@ -1,7 +1,8 @@
 from fastapi import FastAPI, Request
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
-from api.routes import flows, nodes
+from api.routes import flows, nodes, appstate
+import os
 import time
 import logging
 
@@ -9,6 +10,11 @@ import logging
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
+# Detect execution environment and log it at startup
+_ENV = os.environ.get("M8FLOW_ENV", "venv")
+_ENV_LABEL = "Containerized (Docker — Warm)" if _ENV == "docker" else "Python venv"
+logger.info("[SYSTEM] Environment: %s", _ENV_LABEL)
+
 app = FastAPI(title="M8Flow — Code-Driven ML Pipeline Builder")
 
 # CORS configuration
@@ -49,11 +55,17 @@ async def add_process_time_header(request: Request, call_next):
     logger.info(f"Path: {request.url.path} | Time: {process_time:.4f}s")
     return response
 
-app.include_router(flows.router, prefix="/api/flows", tags=["Flows"])
-app.include_router(nodes.router, prefix="/api/nodes", tags=["Nodes"])
+app.include_router(flows.router,    prefix="/api/flows",    tags=["Flows"])
+app.include_router(nodes.router,    prefix="/api/nodes",    tags=["Nodes"])
+app.include_router(appstate.router, prefix="/api/app/state", tags=["AppState"])
 
 
 @app.get("/api/health")
 def health_check():
-    return {"status": "ok"}
+    from config import config
+    return {
+        "status": "ok",
+        "environment": _ENV_LABEL,
+        "server_key_configured": bool(config.OPENROUTER_API_KEY),
+    }
 

package/bundled/backend/requirements.txt

@@ -1,14 +1,35 @@
+# ── Core API ─────────────────────────────────────────────────────────────────
 fastapi
 uvicorn[standard]
-scikit-learn
-joblib
-pandas
+python-multipart
 pydantic
 python-dotenv
-numpy
 httpx
+
+# ── Data ─────────────────────────────────────────────────────────────────────
+pandas
+numpy
+scipy
+joblib
+
+# ── ML ───────────────────────────────────────────────────────────────────────
+scikit-learn
+xgboost
+lightgbm
+statsmodels
+
+# ── Imbalanced learning ───────────────────────────────────────────────────────
+imbalanced-learn
+
+# ── Explainability ────────────────────────────────────────────────────────────
+shap
+
+# ── Dimensionality reduction ──────────────────────────────────────────────────
+umap-learn
+
+# ── Visualisation (server-side, not imported at runtime) ─────────────────────
 matplotlib
 seaborn
-scipy
+
+# ── Legacy / compat ───────────────────────────────────────────────────────────
 openai
-python-multipart