@vespermcp/mcp-server 1.2.20 → 1.2.22

package/scripts/wizard.js

@@ -10,6 +10,9 @@ const path = require('path');
 const os = require('os');
 const crypto = require('crypto');
 const { execSync, spawnSync } = require('child_process');
+const http = require('http');
+const https = require('https');
+const readline = require('readline');
 
 // ── Paths ────────────────────────────────────────────────────
 const HOME = os.homedir();
@@ -54,6 +57,271 @@ function yellow(text) { return `\x1b[33m${text}\x1b[0m`; }
 function red(text) { return `\x1b[31m${text}\x1b[0m`; }
 function magenta(text) { return `\x1b[35m${text}\x1b[0m`; }
 
+// ── Vesper API URL resolution ────────────────────────────────
+const VESPER_API_URL = process.env.VESPER_API_URL || '';
+const DEFAULT_VESPER_API_CANDIDATES = [
+  'https://getvesper.dev',
+  'http://localhost:3000',
+  'http://127.0.0.1:3000',
+];
+
+// ── Device Auth Helpers ──────────────────────────────────────
+function httpJson(method, url, body) {
+  return new Promise((resolve, reject) => {
+    const parsed = new URL(url);
+    const lib = parsed.protocol === 'https:' ? https : http;
+    const opts = {
+      method,
+      hostname: parsed.hostname,
+      port: parsed.port || (parsed.protocol === 'https:' ? 443 : 80),
+      path: parsed.pathname + parsed.search,
+      headers: { 'Content-Type': 'application/json' },
+    };
+    const req = lib.request(opts, (res) => {
+      let data = '';
+      res.on('data', (chunk) => (data += chunk));
+      res.on('end', () => {
+        try { resolve({ status: res.statusCode, body: JSON.parse(data) }); }
+        catch { resolve({ status: res.statusCode, body: data }); }
+      });
+    });
+    req.on('error', reject);
+    if (body) req.write(JSON.stringify(body));
+    req.end();
+  });
+}
+
+async function probeDeviceAuth(baseUrl) {
+  try {
+    const res = await httpJson('POST', `${baseUrl}/api/auth/device/start`);
+    if (res.status === 201 && !!res.body && !!res.body.code) {
+      return { baseUrl, status: 'ready', response: res.body };
+    }
+
+    if (res.status === 503 && res.body && res.body.requiresSetup) {
+      return {
+        baseUrl,
+        status: 'setup-required',
+        response: res.body,
+        message: res.body.error || 'Auth storage is not initialized.',
+      };
+    }
+
+    return {
+      baseUrl,
+      status: 'unreachable',
+      response: res.body,
+      message: typeof res.body === 'string' ? res.body : JSON.stringify(res.body),
+    };
+  } catch (error) {
+    return {
+      baseUrl,
+      status: 'unreachable',
+      message: error && error.message ? error.message : 'Request failed',
+    };
+  }
+}
+
+async function resolveVesperApiBaseUrl() {
+  const candidates = VESPER_API_URL
+    ? [VESPER_API_URL]
+    : DEFAULT_VESPER_API_CANDIDATES;
+
+  let setupRequiredProbe = null;
+
+  for (const candidate of candidates) {
+    const probe = await probeDeviceAuth(candidate);
+    if (probe.status === 'ready') {
+      return probe;
+    }
+
+    if (!setupRequiredProbe && probe.status === 'setup-required') {
+      setupRequiredProbe = probe;
+    }
+  }
+
+  return setupRequiredProbe;
+}
+
+function openBrowser(url) {
+  try {
+    if (process.platform === 'win32') {
+      spawnSync('cmd', ['/c', 'start', '', url], { stdio: 'ignore' });
+    } else if (process.platform === 'darwin') {
+      spawnSync('open', [url], { stdio: 'ignore' });
+    } else {
+      spawnSync('xdg-open', [url], { stdio: 'ignore' });
+    }
+  } catch { /* browser open is best-effort */ }
+}
+
+function askYesNo(question) {
+  return new Promise((resolve) => {
+    const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+    rl.question(`  ${question} ${dim('[Y/n]')} `, (answer) => {
+      rl.close();
+      resolve(!answer || answer.toLowerCase().startsWith('y'));
+    });
+  });
+}
+
+function askInput(question) {
+  return new Promise((resolve) => {
+    const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+    rl.question(`  ${question} `, (answer) => {
+      rl.close();
+      resolve(String(answer || '').trim());
+    });
+  });
+}
+
+async function askChoice(question, choices, defaultValue) {
+  console.log(`  ${question}`);
+  choices.forEach((choice, index) => {
+    console.log(`    ${dim(String(index + 1) + ')')} ${choice.label}`);
+  });
+
+  const prompt = defaultValue ? `${dim('[default: ' + defaultValue + ']')}` : '';
+  const answer = await askInput(`${prompt} ${cyan('→')} Choose an option:`);
+  if (!answer && defaultValue) {
+    return defaultValue;
+  }
+
+  const numeric = Number(answer);
+  if (Number.isFinite(numeric) && numeric >= 1 && numeric <= choices.length) {
+    return choices[numeric - 1].value;
+  }
+
+  const matched = choices.find((choice) => choice.value === answer);
+  return matched ? matched.value : defaultValue;
+}
+
+function isCloudApiKey(value) {
+  return !!value && value.startsWith('vesper_sk_') && !value.startsWith('vesper_sk_local_');
+}
+
+async function promptForManualApiKey() {
+  console.log(`\n  ${cyan('■')} ${bold('Manual API Key')}`);
+  console.log(`  ${dim('Paste a Vesper cloud API key. It will be stored locally in config.toml.\n')}`);
+
+  while (true) {
+    const value = await askInput(`${cyan('→')} Vesper API key:`);
+    if (isCloudApiKey(value)) {
+      return value;
+    }
+    console.log(`  ${yellow('!')} ${yellow('Expected a Vesper key starting with vesper_sk_')}`);
+  }
+}
+
+async function chooseAuthMode(existingKey, existingAuthMode) {
+  const hasExistingKey = !!existingKey;
+  if (hasExistingKey) {
+    console.log(`      ${dim('Current key:')} ${dim(existingKey.slice(0, 24) + '...')}`);
+    console.log(`      ${dim('Current mode:')} ${dim(existingAuthMode || (isCloudApiKey(existingKey) ? 'cloud' : 'local_unified'))}`);
+  }
+
+  const choices = [];
+  choices.push({ value: 'browser', label: 'Sign in through the browser' });
+  choices.push({ value: 'manual', label: 'Provide Vesper API key manually' });
+
+  return await askChoice(`${cyan('→')} How do you want to authenticate Vesper?`, choices, 'browser');
+}
+
+async function deviceAuthFlow() {
+  console.log(`\n  ${cyan('■')} ${bold('Device Authentication')}`);
+  console.log(`  ${dim('Link your CLI to a Vesper account for cloud features\n')}`);
+
+  const resolvedApiBaseUrl = await resolveVesperApiBaseUrl();
+  if (!resolvedApiBaseUrl) {
+    console.log(`  ${red('✗')} ${red('Could not reach any Vesper auth endpoint.')}`);
+    console.log(`      ${dim('Tried:')} ${dim((VESPER_API_URL ? [VESPER_API_URL] : DEFAULT_VESPER_API_CANDIDATES).join(', '))}`);
+    console.log(`      ${dim('If your landing app is running locally, start it on http://localhost:3000 or set VESPER_API_URL.')}`);
+    console.log(`      ${dim('Falling back to manual key entry.\n')}`);
+    return null;
+  }
+
+  if (resolvedApiBaseUrl.status === 'setup-required') {
+    console.log(`  ${yellow('!')} ${yellow('Reached Vesper auth endpoint, but local auth storage is not initialized.')}`);
+    console.log(`      ${dim('Endpoint:')} ${dim(resolvedApiBaseUrl.baseUrl)}`);
+    console.log(`      ${dim('Reason:')} ${dim(resolvedApiBaseUrl.message || 'Apply Supabase migrations first.')}`);
+    console.log(`      ${dim('Run the SQL in supabase/migrations/001_device_auth.sql and 002_rate_limits.sql, then retry.')}`);
+    console.log(`      ${dim('Falling back to manual key entry.\n')}`);
+    return null;
+  }
+
+  console.log(`  ${dim('Auth endpoint:')} ${dim(resolvedApiBaseUrl.baseUrl)}\n`);
+
+  // Step 1: Call /api/auth/device/start
+  process.stdout.write(`  ${dim('Requesting device code...')}`);
+  let startRes;
+  try {
+    startRes = await httpJson('POST', `${resolvedApiBaseUrl.baseUrl}/api/auth/device/start`);
+  } catch (err) {
+    console.log(` ${red('✗')}`);
+    console.log(`      ${red('Could not reach Vesper API at')} ${dim(resolvedApiBaseUrl.baseUrl)}`);
+    console.log(`      ${dim('Falling back to manual key entry.\n')}`);
+    return null;
+  }
+
+  if (startRes.status !== 201 || !startRes.body.code) {
+    console.log(` ${red('✗')}`);
+    console.log(`      ${red('Unexpected response:')} ${dim(JSON.stringify(startRes.body))}`);
+    return null;
+  }
+
+  const { code, loginUrl } = startRes.body;
+  console.log(` ${green('✓')}\n`);
+
+  // Step 2: Display code and open browser
+  console.log(`  ┌───────────────────────────────────────────────┐`);
+  console.log(`  │                                               │`);
+  console.log(`  │   ${bold('Your device code:')}  ${cyan(bold(code))}              │`);
+  console.log(`  │                                               │`);
+  console.log(`  │   ${dim('Open this URL to sign in:')}                   │`);
+  console.log(`  │   ${cyan(loginUrl.padEnd(41))}│`);
+  console.log(`  │                                               │`);
+  console.log(`  └───────────────────────────────────────────────┘\n`);
+
+  openBrowser(loginUrl);
+  console.log(`  ${dim('Browser opened automatically.')}`);
+  console.log(`  ${dim('Waiting for you to sign in...')}\n`);
+
+  // Step 3: Poll until confirmed or expired
+  const POLL_INTERVAL = 3000; // 3 seconds
+  const MAX_POLLS = 200;      // 10 min max (200 × 3s)
+  let polls = 0;
+  const spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
+
+  while (polls < MAX_POLLS) {
+    polls++;
+    const frame = spinner[polls % spinner.length];
+    process.stdout.write(`\r  ${cyan(frame)} Polling... (${polls})`);
+
+    try {
+      const pollRes = await httpJson('GET', `${resolvedApiBaseUrl.baseUrl}/api/auth/device/poll?code=${code}`);
+
+      if (pollRes.body.status === 'confirmed' && pollRes.body.apiKey) {
+        process.stdout.write(`\r  ${green('✓')} Device authenticated!            \n`);
+        console.log(`      ${dim('Email:')} ${pollRes.body.email || 'linked'}`);
+        return pollRes.body.apiKey;
+      }
+
+      if (pollRes.body.status === 'expired') {
+        process.stdout.write(`\r  ${red('✗')} Device code expired.             \n`);
+        console.log(`      ${dim('Run the wizard again to get a new code.')}`);
+        return null;
+      }
+    } catch {
+      // Network hiccup — keep polling
+    }
+
+    await new Promise((r) => setTimeout(r, POLL_INTERVAL));
+  }
+
+  process.stdout.write(`\r  ${red('✗')} Timed out waiting for authentication.\n`);
+  return null;
+}
+
 function printBanner() {
   console.log(`
 ${dim('─────────────────────────────────────────────────')}
@@ -182,21 +450,42 @@ async function main() {
   ensureDir(path.join(VESPER_DIR, 'datasets'));
   console.log(` ${green('✓')}`);
 
-  // ─── Step 2: Generate local API key ────────────────────────
-  process.stdout.write(`  ${dim('[')}${cyan('2/6')}${dim(']')} Generating local API key...`);
+  // ─── Step 2: Authenticate (device flow or local key) ──────
+  console.log(`\n  ${dim('[')}${cyan('2/6')}${dim(']')} Authentication`);
+
   const existing = readToml(CONFIG_TOML);
-  const localKey = existing.api_key || generateLocalKey();
-  const configData = { ...existing, api_key: localKey };
+  let localKey = existing.api_key || '';
+  let authMode = existing.auth_mode || '';
+
+  const authChoice = await chooseAuthMode(localKey, authMode);
+
+  if (authChoice === 'manual') {
+    localKey = await promptForManualApiKey();
+    authMode = 'cloud';
+    console.log(`      ${green('✓')} Cloud API key saved from manual input`);
+  } else if (authChoice === 'browser') {
+    const cloudKey = await deviceAuthFlow();
+    if (cloudKey) {
+      localKey = cloudKey;
+      authMode = 'cloud';
+    } else {
+      console.log(`\n      ${yellow('!')} Browser sign-in did not complete. Falling back to manual key entry.`);
+      localKey = await promptForManualApiKey();
+      authMode = 'cloud';
+    }
+  }
+
+  const configData = { ...existing, api_key: localKey, auth_mode: authMode };
   writeToml(CONFIG_TOML, configData);
-  console.log(` ${green('✓')}`);
-  console.log(`      ${dim('Key:')} ${dim(localKey.slice(0, 20) + '...')}  ${dim('→')} ${dim(CONFIG_TOML)}`);
+  console.log(`      ${dim('Key:')} ${dim(localKey.slice(0, 24) + '...')}  ${dim('→')} ${dim(CONFIG_TOML)}`);
 
   // ─── Step 3: Local vault initialization ────────────────────
   process.stdout.write(`\n  ${dim('[')}${cyan('3/6')}${dim(']')} Initializing local credentials vault...`);
-  configData.auth_mode = configData.auth_mode || 'local_unified';
-  writeToml(CONFIG_TOML, configData);
+  const vaultData = readToml(CONFIG_TOML);
+  if (!vaultData.auth_mode) vaultData.auth_mode = 'local_unified';
+  writeToml(CONFIG_TOML, vaultData);
   console.log(` ${green('✓')}`);
-  console.log(`      ${dim('Mode:')} ${dim('single local Vesper key (no external keys required)')}`);
+  console.log(`      ${dim('Mode:')} ${dim(vaultData.auth_mode === 'cloud' ? 'cloud (linked to Vesper account)' : 'single local Vesper key (no external keys required)')}`);
 
   // ─── Step 4: Install @vespermcp/mcp-server ─────────────────
   console.log(`\n  ${dim('[')}${cyan('4/6')}${dim(']')} Installing Vesper MCP server...`);
@@ -251,19 +540,24 @@ async function main() {
   console.log(`      ${configuredAgents.length > 0 ? green('✓') : yellow('⚠')} MCP agents            ${dim(configuredAgents.length + ' configured')}`);
 
   // ─── Final Summary ─────────────────────────────────────────
+  const finalConfig = readToml(CONFIG_TOML);
+  const isCloud = finalConfig.auth_mode === 'cloud';
   console.log(`
 ${dim('═════════════════════════════════════════════════')}
 
   ${green(bold('✓ Vesper is ready!'))}
 
-  ${bold('Your local API key:')}
-  ${cyan(localKey)}
+  ${bold(isCloud ? 'Your cloud API key:' : 'Your local API key:')}
+  ${cyan(finalConfig.api_key || localKey)}
+
+  ${bold('Auth mode:')}
+  ${dim(isCloud ? '☁  Cloud (linked to Vesper account)' : '🔑 Local-only (key never leaves your machine)')}
 
   ${bold('Config file:')}
   ${dim(CONFIG_TOML)}
 
   ${bold('What just happened:')}
-  ${dim('1.')} Generated a local API key (never leaves your machine)
+  ${dim('1.')} ${isCloud ? 'Linked to your Vesper cloud account' : 'Generated a local API key (never leaves your machine)'}
   ${dim('2.')} Initialized local credentials vault
   ${dim('3.')} Auto-configured MCP for ${configuredAgents.length > 0 ? configuredAgents.join(', ') : 'detected agents'}
   ${dim('4.')} Vesper server ready on stdio transport

package/src/python/asset_downloader_engine.py

@@ -3,9 +3,14 @@ import asyncio
 import json
 import os
 import sys
+import warnings
 from pathlib import Path
 from typing import Any, Dict
 
+# Suppress noisy HF warnings
+warnings.filterwarnings("ignore", message=".*trust_remote_code.*")
+warnings.filterwarnings("ignore", message=".*legacy.*")
+
 CURRENT_DIR = Path(__file__).resolve().parent
 if str(CURRENT_DIR) not in sys.path:
     sys.path.insert(0, str(CURRENT_DIR))
@@ -21,9 +26,15 @@ def _print(payload: Dict[str, Any]) -> None:
 async def _run_download(args: argparse.Namespace) -> Dict[str, Any]:
     payload = json.loads(args.payload)
     output_root = payload.get("output_root") or str(Path.home() / ".vesper" / "data" / "assets")
+    output_dir = payload.get("output_dir")
     workers = int(payload.get("workers") or 8)
     recipes_dir = payload.get("recipes_dir")
 
+    # Auto-set HF token from payload if provided
+    token = payload.get("token") or payload.get("hf_token")
+    if token:
+        os.environ["HF_TOKEN"] = str(token)
+
     downloader = AssetDownloader(output_root=output_root, workers=workers, recipes_dir=recipes_dir)
 
     result = await downloader.download_assets(
@@ -33,6 +44,7 @@ async def _run_download(args: argparse.Namespace) -> Dict[str, Any]:
         kaggle_ref=payload.get("kaggle_ref"),
         urls=payload.get("urls"),
         output_format=payload.get("output_format", "webdataset"),
+        output_dir=str(output_dir) if output_dir else None,
         max_items=payload.get("max_items"),
         image_column=payload.get("image_column"),
     )
@@ -66,7 +78,16 @@ def main() -> None:
 
         _print({"ok": False, "error": f"Unknown action: {args.action}"})
     except Exception as e:
-        _print({"ok": False, "error": str(e)})
+        error_msg = str(e)
+        # Provide actionable error messages
+        if "401" in error_msg or "403" in error_msg or "Unauthorized" in error_msg:
+            error_msg = (
+                "Authentication required. This dataset may be gated/private. "
+                "Use configure_keys tool to set HF_TOKEN, then retry."
+            )
+        elif "No image column" in error_msg:
+            error_msg += " Hint: specify image_column parameter with the name of the column containing images."
+        _print({"ok": False, "error": error_msg})
 
 
 if __name__ == "__main__":

package/src/python/convert_engine.py

@@ -0,0 +1,92 @@
+"""
+Convert a dataset file between formats (CSV, Parquet, JSON, JSONL).
+Usage: convert_engine.py <input_path> <output_path>
+Outputs JSON: {"ok": true, "output_path": "...", "rows": N, "columns": N} or {"ok": false, "error": "..."}
+"""
+import sys
+import json
+import os
+
+try:
+    import polars as pl
+except Exception:
+    print(json.dumps({"ok": False, "error": "polars is required. Install with: pip install polars"}))
+    sys.exit(1)
+
+
+def _load(src: str) -> pl.DataFrame:
+    ext = os.path.splitext(src)[1].lower()
+    if ext == ".csv":
+        return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+    if ext in (".tsv", ".tab"):
+        return pl.read_csv(src, separator="\t", ignore_errors=True, infer_schema_length=10000)
+    if ext in (".parquet", ".pq"):
+        return pl.read_parquet(src)
+    if ext in (".feather", ".ftr", ".arrow", ".ipc"):
+        return pl.read_ipc(src)
+    if ext in (".jsonl", ".ndjson"):
+        return pl.read_ndjson(src)
+    if ext == ".json":
+        raw = open(src, "r", encoding="utf-8").read().strip()
+        if raw.startswith("["):
+            return pl.read_json(src)
+        if "\n" in raw and raw.split("\n")[0].strip().startswith("{"):
+            return pl.read_ndjson(src)
+        obj = json.loads(raw)
+        if isinstance(obj, dict):
+            for key in ("data", "rows", "items", "records", "results", "entries", "samples"):
+                if key in obj and isinstance(obj[key], list):
+                    return pl.DataFrame(obj[key])
+            for v in obj.values():
+                if isinstance(v, list) and len(v) > 0 and isinstance(v[0], dict):
+                    return pl.DataFrame(v)
+        return pl.read_json(src)
+    # Fallback: try csv
+    return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
+
+
+def _write(df: pl.DataFrame, dst: str) -> None:
+    ext = os.path.splitext(dst)[1].lower()
+    os.makedirs(os.path.dirname(dst) or ".", exist_ok=True)
+    if ext in (".parquet", ".pq"):
+        df.write_parquet(dst)
+    elif ext == ".csv":
+        df.write_csv(dst)
+    elif ext == ".json":
+        df.write_json(dst, row_oriented=True)
+    elif ext in (".jsonl", ".ndjson"):
+        df.write_ndjson(dst)
+    else:
+        raise ValueError(f"Unsupported output format: {ext}")
+
+
+def main():
+    if len(sys.argv) < 3:
+        print(json.dumps({"ok": False, "error": "Usage: convert_engine.py <input> <output>"}))
+        sys.exit(1)
+
+    input_path = sys.argv[1]
+    output_path = sys.argv[2]
+
+    if not os.path.exists(input_path):
+        print(json.dumps({"ok": False, "error": f"File not found: {input_path}"}))
+        sys.exit(1)
+
+    try:
+        df = _load(input_path)
+        _write(df, output_path)
+        size_mb = round(os.path.getsize(output_path) / (1024 * 1024), 2)
+        print(json.dumps({
+            "ok": True,
+            "output_path": output_path,
+            "rows": df.height,
+            "columns": df.width,
+            "size_mb": size_mb,
+        }))
+    except Exception as e:
+        print(json.dumps({"ok": False, "error": str(e)}))
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/src/python/export_engine.py

@@ -50,6 +50,51 @@ def _load(file_path: str, options: dict) -> pl.DataFrame:
         df = pl.read_ipc(file_path)
     elif ext == ".jsonl":
         df = pl.read_ndjson(file_path)
+    elif ext == ".json":
+        # Auto-detect: array-of-objects vs NDJSON vs nested structures
+        try:
+            import json as _json
+            with open(file_path, "r", encoding="utf-8", errors="ignore") as fh:
+                raw_text = fh.read(512)  # peek
+            stripped = raw_text.lstrip()
+            if stripped.startswith("["):
+                # Array of objects — standard JSON
+                with open(file_path, "r", encoding="utf-8", errors="ignore") as fh:
+                    data = _json.load(fh)
+                if isinstance(data, list) and len(data) > 0:
+                    df = pl.DataFrame(data)
+                else:
+                    raise ValueError("JSON file is empty or not an array of objects")
+            elif stripped.startswith("{"):
+                # Could be NDJSON or a single object wrapping rows
+                try:
+                    df = pl.read_ndjson(file_path)
+                except Exception:
+                    with open(file_path, "r", encoding="utf-8", errors="ignore") as fh:
+                        data = _json.load(fh)
+                    # Try common wrapper patterns: {"data": [...]}, {"rows": [...]}, etc.
+                    rows = None
+                    if isinstance(data, dict):
+                        for key in ("data", "rows", "records", "items", "results", "entries"):
+                            if key in data and isinstance(data[key], list):
+                                rows = data[key]
+                                break
+                        if rows is None:
+                            # Last resort: try to use the dict values
+                            rows = [data]
+                    if rows and len(rows) > 0:
+                        df = pl.DataFrame(rows)
+                    else:
+                        raise ValueError("Could not parse JSON structure into tabular data")
+            else:
+                raise ValueError("JSON file does not start with [ or {")
+        except pl.exceptions.ComputeError as ce:
+            raise ValueError(f"Failed to parse JSON: {ce}")
+    elif ext == ".xlsx":
+        try:
+            df = pl.read_excel(file_path)
+        except Exception as e:
+            raise ValueError(f"Failed to read Excel file: {e}")
     else:
         raise ValueError(f"Unsupported input format: {ext}")