ds-agent-cli 0.1.0

package/bin/ds-agent.js

@@ -0,0 +1,451 @@
+#!/usr/bin/env node
+
+"use strict";
+
+const fs = require("fs");
+const os = require("os");
+const path = require("path");
+const { spawnSync } = require("child_process");
+
+const pkg = require("../package.json");
+const projectRoot = path.resolve(__dirname, "..");
+const appHome = process.env.DS_AGENT_HOME || path.join(os.homedir(), ".ds-agent");
+const venvDir = path.join(appHome, "venv");
+const stateFile = path.join(appHome, "install-state.json");
+const requirementsFile = path.join(projectRoot, "requirements.txt");
+const cliScript = path.join(projectRoot, "src", "cli.py");
+const INSTALL_SCHEMA_VERSION = 3;
+
+const HEAVY_GROUPS = ["runtime", "data", "frame", "plot", "forecast", "nlp", "bi", "ml", "report", "automl"];
+
+const COMMAND_GROUPS = {
+  quickstart: [],
+  analyze: HEAVY_GROUPS,
+  pipeline: HEAVY_GROUPS,
+  chat: HEAVY_GROUPS,
+  "cache-stats": HEAVY_GROUPS,
+  "clear-cache": HEAVY_GROUPS,
+  tune: ["data", "frame", "ml", "plot", "automl"],
+  train: ["data", "frame", "ml", "plot"],
+  clean: ["data", "frame"],
+  profile: ["data", "frame"],
+  eda: ["data", "frame", "report", "plot"],
+  report: ["data", "report"],
+  compare: ["data", "frame", "report", "plot"],
+  bi: ["data", "frame", "bi", "plot"],
+  plot: ["data", "frame", "plot"],
+  forecast: ["data", "forecast", "plot"],
+  nlp: ["data", "nlp"],
+  sessions: [],
+};
+
+const GROUP_DEFS = {
+  core: {
+    version: 2,
+    packages: [
+      "typer>=0.12.3,<0.16.0",
+      "click>=8.1.7,<8.2.0",
+      "rich==13.7.0",
+      "python-dotenv==1.0.0",
+      "pydantic>=2.11.7",
+      "setuptools<81",
+    ],
+  },
+  data: {
+    version: 2,
+    packages: [
+      "numpy>=1.26.0,<2.3.0",
+      "pandas>=2.2.0,<3.0.0",
+    ],
+  },
+  frame: {
+    version: 1,
+    packages: [
+      "polars>=0.20.3",
+      "pyarrow>=14.0.1",
+      "duckdb>=0.10.0",
+      "joblib==1.3.2",
+    ],
+  },
+  report: {
+    version: 1,
+    packages: ["ydata-profiling>=4.17.0", "sweetviz>=2.3.0"],
+  },
+  plot: {
+    version: 1,
+    packages: ["matplotlib>=3.8.2", "seaborn>=0.13.1", "plotly>=5.18.0"],
+  },
+  forecast: {
+    version: 1,
+    packages: ["prophet>=1.1.5", "holidays>=0.40", "pmdarima>=2.0", "statsmodels>=0.14.1"],
+  },
+  nlp: {
+    version: 1,
+    packages: [
+      "textblob>=0.17.1",
+      "vaderSentiment>=3.3.2",
+      "sentence-transformers>=2.2.2",
+      "tiktoken>=0.5.2",
+    ],
+  },
+  bi: {
+    version: 1,
+    packages: ["scipy>=1.11.4", "statsmodels>=0.14.1", "dowhy>=0.11"],
+  },
+  ml: {
+    version: 1,
+    packages: [
+      "scikit-learn>=1.4.0",
+      "xgboost>=2.0.3",
+      "lightgbm>=4.6.0",
+      "catboost>=1.2.8",
+      "optuna>=3.5.0",
+      "shap>=0.44.1",
+      "imbalanced-learn>=0.12.0",
+      "cleanlab>=2.6",
+      "pandera>=0.18",
+      "boruta>=0.3",
+    ],
+  },
+  runtime: {
+    version: 1,
+    packages: [
+      "groq>=0.13.0",
+      "mistralai>=0.0.7",
+      "supabase==2.28.3",
+      "boto3>=1.28.0",
+      "huggingface_hub>=0.20.0",
+      "fastapi>=0.109.0",
+      "uvicorn>=0.25.0",
+      "python-multipart>=0.0.6",
+      "google-cloud-bigquery==3.14.1",
+      "google-cloud-storage==2.14.0",
+      "google-auth==2.25.2",
+      "google-generativeai==0.3.2",
+      "lime==0.2.0.1",
+      "fairlearn==0.10.0",
+      "evidently>=0.4.0",
+      "dtreeviz>=2.2",
+      "Pillow>=11.0.0",
+    ],
+  },
+  automl: {
+    version: 1,
+    packages: ["autogluon.tabular>=1.2", "autogluon.timeseries>=1.2", "holidays>=0.40"],
+  },
+};
+
+const GROUP_INSTALL_ORDER = ["core", "runtime", "data", "frame", "plot", "forecast", "nlp", "bi", "ml", "report", "automl"];
+
+function log(msg) {
+  process.stderr.write(`${msg}\n`);
+}
+
+function pipEnv() {
+  return {
+    ...process.env,
+    PIP_PROGRESS_BAR: "off",
+    PIP_DISABLE_PIP_VERSION_CHECK: "1",
+  };
+}
+
+function runCommand(command, args, options = {}) {
+  const result = spawnSync(command, args, {
+    stdio: "inherit",
+    ...options,
+  });
+
+  if (result.error) {
+    throw result.error;
+  }
+
+  if (result.status !== 0) {
+    throw new Error(`Command failed: ${command} ${args.join(" ")}`);
+  }
+}
+
+function runCapture(command, args) {
+  const result = spawnSync(command, args, { encoding: "utf8" });
+  if (result.error || result.status !== 0) {
+    return null;
+  }
+  return (result.stdout || result.stderr || "").trim();
+}
+
+function ensureDir(dirPath) {
+  fs.mkdirSync(dirPath, { recursive: true });
+}
+
+function getVenvPython() {
+  if (process.platform === "win32") {
+    return path.join(venvDir, "Scripts", "python.exe");
+  }
+  return path.join(venvDir, "bin", "python");
+}
+
+function detectPython() {
+  const candidates = [];
+  if (process.env.DS_AGENT_PYTHON) {
+    candidates.push(process.env.DS_AGENT_PYTHON);
+  }
+  candidates.push("python3.11", "python3.12", "python3.10", "python3.13", "python3", "python");
+
+  function parseVersion(text) {
+    const match = String(text || "").match(/Python\s+(\d+)\.(\d+)\.(\d+)/i);
+    if (!match) {
+      return null;
+    }
+    return {
+      major: Number(match[1]),
+      minor: Number(match[2]),
+      patch: Number(match[3]),
+    };
+  }
+
+  function isSupported(version) {
+    if (!version) {
+      return false;
+    }
+    if (version.major !== 3) {
+      return false;
+    }
+    // AutoGluon and other pinned dependencies currently support 3.10-3.13 best.
+    return version.minor >= 10 && version.minor <= 13;
+  }
+
+  for (const candidate of candidates) {
+    const out = runCapture(candidate, ["--version"]);
+    const parsed = parseVersion(out);
+    if (parsed && isSupported(parsed)) {
+      return {
+        command: candidate,
+        version: `${parsed.major}.${parsed.minor}.${parsed.patch}`,
+      };
+    }
+  }
+
+  return null;
+}
+
+function boolEnv(name) {
+  const raw = String(process.env[name] || "").toLowerCase().trim();
+  return raw === "1" || raw === "true" || raw === "yes" || raw === "on";
+}
+
+function detectCommand(args) {
+  for (const arg of args) {
+    if (!arg || arg === "--") {
+      continue;
+    }
+    if (arg.startsWith("-")) {
+      continue;
+    }
+    return arg;
+  }
+  return null;
+}
+
+function requestedGroups(args) {
+  const command = detectCommand(args);
+
+  if (boolEnv("DS_AGENT_INSTALL_ALL")) {
+    return { command, groups: [...GROUP_INSTALL_ORDER] };
+  }
+
+  const groups = new Set(["core"]);
+  const mapped = COMMAND_GROUPS[command] || [];
+  for (const groupName of mapped) {
+    groups.add(groupName);
+  }
+
+  // For unknown commands or typos, keep installs minimal (core only).
+  // This prevents long installs before Click/Typer returns a command error.
+
+  return {
+    command,
+    groups: GROUP_INSTALL_ORDER.filter((name) => groups.has(name)),
+  };
+}
+
+function readState() {
+  try {
+    if (!fs.existsSync(stateFile)) {
+      return null;
+    }
+    const raw = fs.readFileSync(stateFile, "utf8");
+    return JSON.parse(raw);
+  } catch (_err) {
+    return null;
+  }
+}
+
+function writeState(state) {
+  fs.writeFileSync(stateFile, JSON.stringify(state, null, 2), "utf8");
+}
+
+function installPackageSet(venvPython, packages, label) {
+  if (!packages || packages.length === 0) {
+    return;
+  }
+
+  log(`[ds-agent] Installing ${label} dependencies...`);
+  runCommand(
+    venvPython,
+    [
+      "-m",
+      "pip",
+      "install",
+      "--disable-pip-version-check",
+      "--progress-bar",
+      "off",
+      "--upgrade-strategy",
+      "only-if-needed",
+      "--no-input",
+      ...packages,
+    ],
+    { env: pipEnv() }
+  );
+}
+
+function ensureGroupInstalled(venvPython, state, groupName) {
+  const groupDef = GROUP_DEFS[groupName];
+  if (!groupDef) {
+    return;
+  }
+
+  if ((state.installedGroups || {})[groupName] === groupDef.version) {
+    return;
+  }
+
+  installPackageSet(venvPython, groupDef.packages, groupName);
+  state.installedGroups[groupName] = groupDef.version;
+  state.lastInstalledAt = new Date().toISOString();
+  writeState(state);
+}
+
+function bootstrap(args) {
+  ensureDir(appHome);
+
+  const python = detectPython();
+  if (!python) {
+    throw new Error(
+      "No compatible Python found. Install Python 3.10-3.13 or set DS_AGENT_PYTHON to a compatible executable."
+    );
+  }
+
+  const venvPython = getVenvPython();
+
+  if (!fs.existsSync(venvPython)) {
+    log("[ds-agent] Creating Python virtual environment...");
+    runCommand(python.command, ["-m", "venv", venvDir]);
+  }
+
+  let state = readState();
+  const hardResetNeeded =
+    !state ||
+    state.installSchemaVersion !== INSTALL_SCHEMA_VERSION ||
+    state.packageVersion !== pkg.version ||
+    state.pythonCommand !== python.command ||
+    state.pythonVersion !== python.version;
+
+  if (hardResetNeeded) {
+    state = {
+      installSchemaVersion: INSTALL_SCHEMA_VERSION,
+      packageVersion: pkg.version,
+      pythonCommand: python.command,
+      pythonVersion: python.version,
+      requirementsMtime: fs.existsSync(requirementsFile) ? fs.statSync(requirementsFile).mtimeMs : null,
+      installedGroups: {},
+      updatedAt: new Date().toISOString(),
+    };
+    writeState(state);
+  }
+
+  if (!(state.installedGroups || {}).core) {
+    log("[ds-agent] Preparing Python toolchain...");
+    runCommand(
+      venvPython,
+      [
+        "-m",
+        "pip",
+        "install",
+        "--upgrade",
+        "--disable-pip-version-check",
+        "--progress-bar",
+        "off",
+        "pip",
+        "setuptools<81",
+        "wheel",
+      ],
+      { env: pipEnv() }
+    );
+    ensureGroupInstalled(venvPython, state, "core");
+  }
+
+  const request = requestedGroups(args);
+  for (const groupName of request.groups) {
+    ensureGroupInstalled(venvPython, state, groupName);
+  }
+
+  if (boolEnv("DS_AGENT_INSTALL_ALL")) {
+    log("[ds-agent] DS_AGENT_INSTALL_ALL is enabled, syncing full requirements.txt...");
+    runCommand(
+      venvPython,
+      [
+        "-m",
+        "pip",
+        "install",
+        "--disable-pip-version-check",
+        "--progress-bar",
+        "off",
+        "--upgrade-strategy",
+        "only-if-needed",
+        "--no-input",
+        "-r",
+        requirementsFile,
+      ],
+      { env: pipEnv() }
+    );
+  }
+
+  state.updatedAt = new Date().toISOString();
+  writeState(state);
+
+  return venvPython;
+}
+
+function runCli() {
+  const args = process.argv.slice(2);
+  const venvPython = bootstrap(args);
+
+  const disableTqdm = !boolEnv("DS_AGENT_VERBOSE_PROGRESS");
+
+  const env = {
+    ...process.env,
+    PIP_PROGRESS_BAR: process.env.PIP_PROGRESS_BAR || "off",
+    TQDM_DISABLE: disableTqdm ? "1" : process.env.TQDM_DISABLE || "0",
+    HF_HUB_DISABLE_PROGRESS_BARS: disableTqdm ? "1" : process.env.HF_HUB_DISABLE_PROGRESS_BARS || "0",
+    TOKENIZERS_PARALLELISM: process.env.TOKENIZERS_PARALLELISM || "false",
+    PYTHONPATH: process.env.PYTHONPATH
+      ? `${projectRoot}${path.delimiter}${process.env.PYTHONPATH}`
+      : projectRoot,
+  };
+
+  const result = spawnSync(venvPython, [cliScript, ...args], {
+    stdio: "inherit",
+    env,
+  });
+
+  if (result.error) {
+    throw result.error;
+  }
+
+  process.exit(result.status || 0);
+}
+
+try {
+  runCli();
+} catch (err) {
+  log(`[ds-agent] ${err.message || String(err)}`);
+  process.exit(1);
+}

package/ds_agent/__init__.py

@@ -0,0 +1,8 @@
+from pathlib import Path
+
+__version__ = "0.1.0"
+
+# Allow ds_agent.* imports to resolve modules currently stored under src/.
+_src_dir = Path(__file__).resolve().parent.parent / "src"
+if _src_dir.exists():
+    __path__.append(str(_src_dir))

package/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "ds-agent-cli",
+  "version": "0.1.0",
+  "description": "DS-Agent CLI wrapper for npm global install",
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "bin": {
+    "ds-agent": "bin/ds-agent.js"
+  },
+  "files": [
+    "bin/ds-agent.js",
+    "src/**/*.py",
+    "ds_agent/__init__.py",
+    "requirements.txt",
+    "setup.py"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "keywords": [
+    "data-science",
+    "cli",
+    "python",
+    "ml"
+  ]
+}

package/requirements.txt

@@ -0,0 +1,126 @@
+# Core Dependencies
+groq>=0.13.0  # Updated for httpx compatibility
+mistralai>=0.0.7  # Mistral AI - 1B tokens/month (corrected version)
+python-dotenv==1.0.0
+
+# Data Processing
+polars>=0.20.3
+duckdb>=0.10.0
+pyarrow>=14.0.1
+pandas>=2.2.0  # Updated for Python 3.13 compatibility
+
+# Machine Learning
+scikit-learn>=1.4.0
+xgboost>=2.0.3
+lightgbm>=4.6.0
+catboost>=1.2.8
+optuna>=3.5.0
+
+# AutoGluon AutoML (modular install - only tabular + timeseries)
+autogluon.tabular>=1.2
+autogluon.timeseries>=1.2
+holidays>=0.40  # Holiday calendar for time series covariates
+
+# Explainability
+shap>=0.44.1
+
+# Advanced ML Tools
+imbalanced-learn>=0.12.0
+
+# Statistical Analysis
+scipy>=1.11.4
+statsmodels>=0.14.1
+
+# Visualization
+matplotlib>=3.8.2
+seaborn>=0.13.1
+plotly>=5.18.0  # Interactive visualizations
+
+# EDA Report Generation
+ydata-profiling>=4.17.0  # Comprehensive automated EDA reports with Python 3.13 compatibility
+sweetviz>=2.3.0  # Interactive EDA with comparison support
+
+# User Interface
+# gradio>=5.49.1
+
+# REST API (Cloud Run)
+fastapi>=0.109.0
+uvicorn>=0.25.0
+python-multipart>=0.0.6  # For file uploads
+
+# Text Processing
+textblob>=0.17.1
+vaderSentiment>=3.3.2  # Rule-based sentiment analysis (fast, no GPU needed)
+
+# Time Series Forecasting
+prophet>=1.1.5
+holidays>=0.38
+pmdarima>=2.0  # Auto ARIMA (auto_arima) for optimal order selection
+
+# MLOps & Explainability
+lime==0.2.0.1
+fairlearn==0.10.0
+evidently>=0.4.0  # Production-grade data drift detection & monitoring
+dtreeviz>=2.2  # Decision tree visualization
+
+# NLP & Semantic Layer (REQUIRED for column understanding and agent routing)
+sentence-transformers>=2.2.2  # For semantic column embeddings and agent routing
+tiktoken>=0.5.2  # For accurate token counting in budget management
+
+# Advanced NLP (Optional - Uncomment for advanced NLP tools)
+# These are optional but recommended for full NLP capabilities
+# spacy==3.7.2  # For named entity recognition (perform_named_entity_recognition)
+# transformers==4.35.2  # For transformer-based sentiment & topic modeling
+# bertopic==0.16.0  # For advanced topic modeling
+
+# Computer Vision (Optional - Uncomment for CV tools)
+# These are optional but recommended for full CV capabilities
+# torch==2.1.0  # For CNN-based image feature extraction
+# torchvision==0.16.0  # For pre-trained models (ResNet, EfficientNet, VGG)
+Pillow>=11.0.0  # For basic image processing
+#opencv-python==4.8.1  # For advanced image processing & color features
+
+# Business Intelligence (Optional - Uncomment for advanced BI tools)
+# These are optional but add specialized capabilities
+# lifetimes==0.11.3  # For customer lifetime value modeling
+# econml==0.15.0  # For advanced causal inference
+dowhy>=0.11  # Formal causal inference with DAG-based reasoning
+
+# Data Quality & Validation
+cleanlab>=2.6  # Label error detection using confident learning
+pandera>=0.18  # Schema-based DataFrame validation
+boruta>=0.3  # All-relevant feature selection (BorutaPy)
+
+# CLI & UI
+typer>=0.12.3
+rich==13.7.0
+tqdm==4.66.1
+
+# Utilities
+pydantic>=2.11.7
+joblib==1.3.2
+
+# Google Cloud Integration
+google-cloud-bigquery==3.14.1
+google-cloud-storage==2.14.0  # For GCS artifact storage
+google-auth==2.25.2
+google-generativeai==0.3.2  # For Gemini LLM support
+
+# Cloudflare R2 Storage (S3-compatible)
+boto3>=1.28.0  # For R2 file storage
+
+# HuggingFace Storage Integration
+huggingface_hub>=0.20.0  # For storing user artifacts on HuggingFace
+
+# Supabase Backend
+supabase==2.28.3  # For user file metadata
+
+# Testing
+pytest==7.4.3
+pytest-mock==3.12.0
+pytest-cov==4.1.0
+
+# Development
+black==23.12.1
+flake8==7.0.0
+mypy==1.8.0

package/setup.py

@@ -0,0 +1,35 @@
+from setuptools import find_packages, setup
+
+
+setup(
+    name="ds-agent",
+    version="0.1.0",
+    description="Data Science Agent CLI",
+    packages=find_packages(),
+    include_package_data=True,
+    install_requires=[
+        "typer",
+        "rich",
+        "polars",
+        "pandas",
+        "matplotlib",
+        "seaborn",
+        "plotly",
+        "groq",
+        "python-dotenv",
+        "gradio",
+        "ydata-profiling",
+        "sweetviz",
+        "scikit-learn",
+        "optuna",
+        "xgboost",
+        "prophet",
+        "statsmodels",
+        "imbalanced-learn",
+    ],
+    entry_points={
+        "console_scripts": [
+            "ds-agent=ds_agent.cli:main",
+        ]
+    },
+)

package/src/__init__.py

@@ -0,0 +1,7 @@
+"""Data Science Copilot - AI-powered data science automation."""
+
+__version__ = "0.1.0"
+
+from .orchestrator import DataScienceCopilot
+
+__all__ = ["DataScienceCopilot"]