@vespermcp/setup 3.0.1 → 3.1.1

package/README.md

@@ -2,6 +2,8 @@
 
 Interactive setup for the Vesper MCP stack: link your [getvesper.dev](https://getvesper.dev) account, store your API key locally, and auto-write MCP config for Cursor, Claude, VS Code, Codex, and Gemini CLI.
 
+The wizard also asks whether you want **web URL extraction** (`vesper.extract_web`). If you opt in and have **Docker**, it runs **`docker pull`** / **`docker run`** for the official [Crawl4AI](https://hub.docker.com/r/unclecode/crawl4ai) image and sets `VESPER_CRAWL4AI_URL` in your MCP env — no separate web search required.
+
 ## Run
 
 ```bash

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vespermcp/setup",
-  "version": "3.0.1",
+  "version": "3.1.1",
   "description": "Zero-friction setup wizard for Vesper — local MCP server, unified dataset API, and agent auto-config in 60 seconds",
   "bin": {
     "setup": "wizard.js"

package/wizard.js

@@ -388,13 +388,18 @@ function buildMcpServerEntry() {
   const npxCmd = IS_WIN ? 'npx.cmd' : 'npx';
   const state = readToml(CONFIG_TOML);
   const apiKey = String(state.api_key || '').trim();
+  const crawl4aiUrl = String(state.crawl4ai_url || '').trim();
+  const env = {
+    VESPER_API_URL: getMcpVesperApiUrl(),
+    VESPER_API_KEY: apiKey || 'your-key-from-getvesper.dev',
+  };
+  if (crawl4aiUrl) {
+    env.VESPER_CRAWL4AI_URL = crawl4aiUrl;
+  }
   return {
     command: npxCmd,
     args: ['-y', '-p', '@vespermcp/mcp-server@latest', 'vespermcp'],
-    env: {
-      VESPER_API_URL: getMcpVesperApiUrl(),
-      VESPER_API_KEY: apiKey || 'your-key-from-getvesper.dev',
-    },
+    env,
   };
 }
 
@@ -405,13 +410,17 @@ function escapeTomlDoubleQuoted(value) {
 function upsertTomlMcpVesperBlock(content, serverEntry) {
   const key = escapeTomlDoubleQuoted(serverEntry.env.VESPER_API_KEY);
   const url = escapeTomlDoubleQuoted(serverEntry.env.VESPER_API_URL);
+  const crawlLine = serverEntry.env.VESPER_CRAWL4AI_URL
+    ? `VESPER_CRAWL4AI_URL = "${escapeTomlDoubleQuoted(serverEntry.env.VESPER_CRAWL4AI_URL)}"\n`
+    : '';
   const block =
     `[mcp_servers.vesper]\n` +
     `command = "${serverEntry.command}"\n` +
     `args = [${serverEntry.args.map((a) => `"${a}"`).join(', ')}]\n\n` +
     `[mcp_servers.vesper.env]\n` +
     `VESPER_API_KEY = "${key}"\n` +
-    `VESPER_API_URL = "${url}"\n`;
+    `VESPER_API_URL = "${url}"\n` +
+    crawlLine;
   const re = /\[mcp_servers\.vesper\][\s\S]*?(?=\n\[|$)/;
   if (re.test(content)) {
     return content.replace(re, block.trim() + '\n');
@@ -512,6 +521,106 @@ async function checkServerHealth() {
   }
 }
 
+const CRAWL4AI_CONTAINER = 'vesper-crawl4ai';
+const CRAWL4AI_DEFAULT_URL = 'http://127.0.0.1:11235';
+
+function printManualCrawl4AiInstructions() {
+  console.log(`\n  ${dim('Run these in a terminal when Docker is available:')}`);
+  console.log(`  ${cyan('docker pull unclecode/crawl4ai')}`);
+  console.log(
+    `  ${cyan(
+      `docker run -d --name ${CRAWL4AI_CONTAINER} -p 11235:11235 --shm-size=1g --restart unless-stopped unclecode/crawl4ai`,
+    )}`,
+  );
+  console.log(`  ${dim('Then set in MCP env:')} ${cyan(`VESPER_CRAWL4AI_URL=${CRAWL4AI_DEFAULT_URL}`)}`);
+  console.log(`  ${dim('Docs:')} https://github.com/vesper/mcp-server/blob/main/docs/web-extraction.md\n`);
+}
+
+function dockerCliWorks() {
+  const r = spawnSync('docker', ['info'], { encoding: 'utf8', timeout: 20000, stdio: ['pipe', 'pipe', 'pipe'] });
+  return r.status === 0;
+}
+
+function refreshMcpConfigsForAgents() {
+  const agents = getAllAgentConfigs();
+  for (const agent of agents) {
+    const dirExists = fs.existsSync(path.dirname(agent.path));
+    const fileExists = fs.existsSync(agent.path);
+    if (fileExists || dirExists) {
+      installMcpToAgent(agent);
+    }
+  }
+}
+
+async function runCrawl4AiDockerSetup() {
+  if (!dockerCliWorks()) {
+    console.log(`  ${yellow('!')} ${yellow('Docker CLI did not respond (is Docker Desktop / the daemon running?)')}`);
+    printManualCrawl4AiInstructions();
+    return false;
+  }
+
+  console.log(`\n  ${dim('Pulling image unclecode/crawl4ai (first run may take a few minutes)...')}`);
+  const pull = spawnSync('docker', ['pull', 'unclecode/crawl4ai'], {
+    stdio: 'inherit',
+    timeout: 600000,
+  });
+  if (pull.error) {
+    console.log(`  ${red('✗')} docker pull failed: ${pull.error.message || pull.error}`);
+    printManualCrawl4AiInstructions();
+    return false;
+  }
+  if (pull.status !== 0) {
+    console.log(`  ${red('✗')} docker pull exited with status ${pull.status}`);
+    printManualCrawl4AiInstructions();
+    return false;
+  }
+
+  const inspect = spawnSync('docker', ['inspect', CRAWL4AI_CONTAINER], {
+    encoding: 'utf8',
+    stdio: ['pipe', 'pipe', 'pipe'],
+  });
+  if (inspect.status === 0) {
+    console.log(`  ${dim('Container')} ${cyan(CRAWL4AI_CONTAINER)} ${dim('exists — starting it...')}`);
+    const start = spawnSync('docker', ['start', CRAWL4AI_CONTAINER], { stdio: 'inherit', timeout: 120000 });
+    if (start.status !== 0) {
+      console.log(`  ${yellow('!')} docker start failed — try: ${cyan(`docker start ${CRAWL4AI_CONTAINER}`)}`);
+    }
+  } else {
+    console.log(`  ${dim('Creating container')} ${cyan(CRAWL4AI_CONTAINER)}...`);
+    const run = spawnSync(
+      'docker',
+      [
+        'run',
+        '-d',
+        '--name',
+        CRAWL4AI_CONTAINER,
+        '-p',
+        '11235:11235',
+        '--shm-size',
+        '1g',
+        '--restart',
+        'unless-stopped',
+        'unclecode/crawl4ai',
+      ],
+      { stdio: 'inherit', timeout: 180000 },
+    );
+    if (run.status !== 0) {
+      console.log(`  ${red('✗')} docker run failed`);
+      printManualCrawl4AiInstructions();
+      return false;
+    }
+  }
+
+  const merged = { ...readToml(CONFIG_TOML), crawl4ai_url: CRAWL4AI_DEFAULT_URL };
+  writeToml(CONFIG_TOML, merged);
+  console.log(`  ${green('✓')} Saved ${dim('crawl4ai_url')} = ${cyan(CRAWL4AI_DEFAULT_URL)} ${dim('→')} ${dim(CONFIG_TOML)}`);
+
+  refreshMcpConfigsForAgents();
+  console.log(`  ${green('✓')} Updated MCP client configs with ${cyan('VESPER_CRAWL4AI_URL')}`);
+  console.log(`  ${dim('Crawl4AI API (after a few seconds):')} ${cyan(`${CRAWL4AI_DEFAULT_URL}/playground`)}`);
+  return true;
+}
+
 // ── Main Wizard ──────────────────────────────────────────────
 async function main() {
   if (!isInteractiveTerminal()) {
@@ -526,7 +635,7 @@ async function main() {
   console.log(`  ${green('→')} Setting up Vesper on ${bold(os.hostname())}\n`);
 
   // ─── Step 1: Create directories ────────────────────────────
-  process.stdout.write(`  ${dim('[')}${cyan('1/6')}${dim(']')} Creating local directories...`);
+  process.stdout.write(`  ${dim('[')}${cyan('1/7')}${dim(']')} Creating local directories...`);
   ensureDir(VESPER_DIR);
   ensureDir(DATA_DIR);
   ensureDir(path.join(DATA_DIR, 'raw'));
@@ -535,7 +644,7 @@ async function main() {
   console.log(` ${green('✓')}`);
 
   // ─── Step 2: Authenticate (device flow or local key) ──────
-  console.log(`\n  ${dim('[')}${cyan('2/6')}${dim(']')} Authentication`);
+  console.log(`\n  ${dim('[')}${cyan('2/7')}${dim(']')} Authentication`);
 
   const existing = readToml(CONFIG_TOML);
   let localKey = existing.api_key || '';
@@ -564,7 +673,7 @@ async function main() {
   console.log(`      ${dim('Key:')} ${dim(localKey.slice(0, 24) + '...')}  ${dim('→')} ${dim(CONFIG_TOML)}`);
 
   // ─── Step 3: Local vault initialization ────────────────────
-  process.stdout.write(`\n  ${dim('[')}${cyan('3/6')}${dim(']')} Initializing local credentials vault...`);
+  process.stdout.write(`\n  ${dim('[')}${cyan('3/7')}${dim(']')} Initializing local credentials vault...`);
   const vaultData = readToml(CONFIG_TOML);
   if (!vaultData.auth_mode) vaultData.auth_mode = 'local_unified';
   writeToml(CONFIG_TOML, vaultData);
@@ -572,7 +681,7 @@ async function main() {
   console.log(`      ${dim('Mode:')} ${dim(vaultData.auth_mode === 'cloud' ? 'cloud (linked to Vesper account)' : 'single local Vesper key (no external keys required)')}`);
 
   // ─── Step 4: Verify @vespermcp/mcp-server command ───────────
-  console.log(`\n  ${dim('[')}${cyan('4/6')}${dim(']')} Verifying Vesper MCP server command...`);
+  console.log(`\n  ${dim('[')}${cyan('4/7')}${dim(']')} Verifying Vesper MCP server command...`);
   try {
     const npmCmd = IS_WIN ? 'npx.cmd' : 'npx';
     const verify = spawnSync(npmCmd, ['-y', '-p', '@vespermcp/mcp-server@latest', 'vespermcp', '--version'], {
@@ -590,7 +699,7 @@ async function main() {
   }
 
   // ─── Step 5: Auto-configure all detected IDEs ──────────────
-  process.stdout.write(`\n  ${dim('[')}${cyan('5/6')}${dim(']')} Configuring coding agents...`);
+  process.stdout.write(`\n  ${dim('[')}${cyan('5/7')}${dim(']')} Configuring coding agents...`);
   const agents = getAllAgentConfigs();
   const configuredAgents = [];
   const skippedAgents = [];
@@ -616,8 +725,27 @@ async function main() {
     console.log(`  └───────────────────────────────────────────────┘`);
   }
 
-  // ─── Step 6: Verify ────────────────────────────────────────
-  console.log(`\n  ${dim('[')}${cyan('6/6')}${dim(']')} Verifying installation...`);
+  // ─── Step 6: Web extraction (Crawl4AI + Docker) ─────────────
+  console.log(`\n  ${dim('[')}${cyan('6/7')}${dim(']')} Web extraction (optional)`);
+  console.log(
+    `  ${dim('Tools like')} ${cyan('vesper.extract_web')} ${dim('use Crawl4AI. The easiest way is Docker on your machine (no extra API keys).')}\n`,
+  );
+
+  const wantWebExtraction = await askYesNo(`${cyan('→')} Enable web URL extraction (Crawl4AI + Docker)?`);
+  if (wantWebExtraction) {
+    const hasDocker = await askYesNo(`${cyan('→')} Do you have Docker installed and running (Docker Desktop / engine)?`);
+    if (hasDocker) {
+      await runCrawl4AiDockerSetup();
+    } else {
+      console.log(`\n  ${dim('Install Docker from https://docs.docker.com/get-docker/ then run the commands below, or re-run this wizard.')}`);
+      printManualCrawl4AiInstructions();
+    }
+  } else {
+    console.log(`  ${dim('Skipped. To enable later, see docs/web-extraction.md')}`);
+  }
+
+  // ─── Step 7: Verify ────────────────────────────────────────
+  console.log(`\n  ${dim('[')}${cyan('7/7')}${dim(']')} Verifying installation...`);
 
   const dbExists = fs.existsSync(path.join(DATA_DIR, 'metadata.db'));
   const vecExists = fs.existsSync(path.join(DATA_DIR, 'vectors.json')) || fs.existsSync(path.join(DATA_DIR, 'vectors.bin'));
@@ -649,7 +777,8 @@ ${dim('════════════════════════
   ${dim('1.')} ${isCloud ? 'Linked to your Vesper cloud account' : 'Generated a local API key (never leaves your machine)'}
   ${dim('2.')} Initialized local credentials vault
   ${dim('3.')} Auto-configured MCP for ${configuredAgents.length > 0 ? configuredAgents.join(', ') : 'detected agents'}
-  ${dim('4.')} Vesper server ready on stdio transport
+  ${dim('4.')} Optional: Crawl4AI for web extraction (${dim('vesper.extract_web')}) — Docker step if you chose it
+  ${dim('5.')} Vesper server ready on stdio transport
 
 ${dim('─────────────────────────────────────────────────')}