crawlio-browser 1.3.0

package/dist/mcp-server/init-TRQTWLAB.js

@@ -0,0 +1,492 @@
+import "./chunk-JSBRDJBE.js";
+
+// src/mcp-server/init.ts
+import { execFileSync, spawn } from "child_process";
+import { existsSync, mkdirSync, writeFileSync, readFileSync, readdirSync, copyFileSync } from "fs";
+import { join, resolve, dirname, sep, basename } from "path";
+import { homedir, platform } from "os";
+import { createServer as createNetServer } from "net";
+import { createInterface } from "readline";
+import { fileURLToPath } from "url";
+var PORTAL_URL = "http://127.0.0.1:3001";
+var HEALTH_URL = `${PORTAL_URL}/health`;
+var MCP_URL = `${PORTAL_URL}/mcp`;
+var HOME = homedir();
+var bold = (s) => `\x1B[1m${s}\x1B[0m`;
+var green = (s) => `\x1B[32m${s}\x1B[0m`;
+var cyan = (s) => `\x1B[36m${s}\x1B[0m`;
+var dim = (s) => `\x1B[2m${s}\x1B[0m`;
+var yellow = (s) => `\x1B[33m${s}\x1B[0m`;
+function parseFlags(argv) {
+  const opts = {
+    portal: false,
+    full: false,
+    dryRun: false,
+    plugin: false,
+    agents: [],
+    yes: false
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg === "--portal") opts.portal = true;
+    else if (arg === "--full") opts.full = true;
+    else if (arg === "--dry-run") opts.dryRun = true;
+    else if (arg === "--plugin") opts.plugin = true;
+    else if (arg === "--yes" || arg === "-y") opts.yes = true;
+    else if (arg === "-a" && i + 1 < argv.length) {
+      opts.agents.push(argv[++i]);
+    }
+  }
+  return opts;
+}
+function buildAddMcpArgs(options) {
+  const args = ["-y", "add-mcp"];
+  if (options.portal) {
+    args.push(MCP_URL);
+  } else if (options.full) {
+    args.push("crawlio-browser --full");
+  } else {
+    args.push("crawlio-browser");
+  }
+  args.push("--name", "crawlio-browser", "--global", "--yes");
+  for (const agent of options.agents) {
+    args.push("-a", agent);
+  }
+  return args;
+}
+function buildStdioEntry(options) {
+  const args = ["-y", "crawlio-browser"];
+  if (options?.full) args.push("--full");
+  return { command: "npx", args };
+}
+function buildPortalEntry() {
+  return { type: "http", url: MCP_URL };
+}
+function isAlreadyConfigured(config) {
+  return "crawlio-browser" in config.mcpServers;
+}
+async function confirm(question, defaultYes = true) {
+  if (!process.stdin.isTTY || !process.stdout.isTTY) return defaultYes;
+  const rl = createInterface({ input: process.stdin, output: process.stdout });
+  const hint = defaultYes ? "[Y/n]" : "[y/N]";
+  return new Promise((resolve2) => {
+    rl.question(`  ${question} ${dim(hint)} `, (answer) => {
+      rl.close();
+      const a = answer.trim().toLowerCase();
+      resolve2(a === "" ? defaultYes : a === "y" || a === "yes");
+    });
+  });
+}
+function findMcpConfig() {
+  const candidates = [
+    join(process.cwd(), ".mcp.json"),
+    join(HOME, ".mcp.json")
+  ];
+  for (const p of candidates) {
+    if (!existsSync(p)) continue;
+    try {
+      const raw = readFileSync(p, "utf-8");
+      const parsed = JSON.parse(raw);
+      if (parsed && typeof parsed === "object" && "mcpServers" in parsed) {
+        return { path: p, config: parsed };
+      }
+    } catch {
+    }
+  }
+  return null;
+}
+async function healthCheck() {
+  try {
+    const res = await fetch(HEALTH_URL);
+    if (!res.ok) return { ok: false };
+    const data = await res.json();
+    return { ok: true, toolCount: data.toolCount, bridgeConnected: data.bridgeConnected };
+  } catch {
+    return { ok: false };
+  }
+}
+async function waitForHealth(retries, delayMs) {
+  for (let i = 0; i < retries; i++) {
+    const result = await healthCheck();
+    if (result.ok) return result;
+    await new Promise((r) => setTimeout(r, delayMs));
+  }
+  return { ok: false };
+}
+function getServerEntryPath() {
+  return resolve(dirname(fileURLToPath(import.meta.url)), "index.js");
+}
+function resolveNodePath() {
+  try {
+    const cmd = platform() === "win32" ? "where" : "which";
+    const result = execFileSync(cmd, ["node"], { encoding: "utf-8", timeout: 5e3 }).trim();
+    const firstLine = result.split("\n")[0].trim();
+    if (firstLine && existsSync(firstLine)) return firstLine;
+  } catch {
+  }
+  return process.execPath;
+}
+function isPortFree(port) {
+  return new Promise((resolve2) => {
+    const srv = createNetServer();
+    srv.once("error", () => resolve2(false));
+    srv.once("listening", () => {
+      srv.close(() => resolve2(true));
+    });
+    srv.listen(port, "127.0.0.1");
+  });
+}
+function generatePlist(nodePath, serverPath) {
+  const logDir = join(HOME, "Library/Logs/Crawlio");
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  <string>com.crawlio.browser</string>
+  <key>ProgramArguments</key>
+  <array>
+    <string>${nodePath}</string>
+    <string>${serverPath}</string>
+    <string>--portal</string>
+  </array>
+  <key>RunAtLoad</key>
+  <true/>
+  <key>KeepAlive</key>
+  <true/>
+  <key>StandardOutPath</key>
+  <string>${logDir}/server.log</string>
+  <key>StandardErrorPath</key>
+  <string>${logDir}/server.err</string>
+  <key>EnvironmentVariables</key>
+  <dict>
+    <key>PATH</key>
+    <string>/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin</string>
+  </dict>
+</dict>
+</plist>`;
+}
+async function ensurePortalRunning(dryRun) {
+  console.log(`  Starting portal server...`);
+  const health = await healthCheck();
+  if (health.ok) {
+    console.log(`    ${green("+")} Server already running on ${PORTAL_URL}`);
+    return;
+  }
+  const serverPath = getServerEntryPath();
+  const nodePath = resolveNodePath();
+  if (dryRun) {
+    console.log(`    ${dim("[dry-run]")} Node path: ${nodePath}`);
+    console.log(`    ${dim("[dry-run]")} Server entry: ${serverPath}`);
+    if (platform() === "darwin") {
+      const plistPath = join(HOME, "Library/LaunchAgents/com.crawlio.browser.plist");
+      console.log(`    ${dim("[dry-run]")} Would write plist to: ${plistPath}`);
+      console.log(`    ${dim("[dry-run]")} Would run: launchctl load ${plistPath}`);
+    } else {
+      console.log(`    ${dim("[dry-run]")} Would spawn detached: ${nodePath} ${serverPath} --portal`);
+    }
+    return;
+  }
+  if (platform() === "darwin") {
+    const plistDir = join(HOME, "Library/LaunchAgents");
+    const plistPath = join(plistDir, "com.crawlio.browser.plist");
+    const logDir = join(HOME, "Library/Logs/Crawlio");
+    mkdirSync(logDir, { recursive: true });
+    mkdirSync(plistDir, { recursive: true });
+    try {
+      execFileSync("launchctl", ["unload", plistPath], { stdio: "ignore" });
+    } catch {
+    }
+    writeFileSync(plistPath, generatePlist(nodePath, serverPath));
+    try {
+      execFileSync("launchctl", ["load", plistPath]);
+    } catch {
+      return startDetachedServer(serverPath, nodePath);
+    }
+    const result = await waitForHealth(5, 1e3);
+    if (result.ok) {
+      console.log(`    ${green("+")} Server running on ${PORTAL_URL}`);
+      console.log(`    ${green("+")} Auto-start on login configured (launchd)`);
+      return;
+    }
+    const portFree = await isPortFree(3001);
+    if (!portFree) {
+      console.log(`    ${yellow("!")} Port 3001 is already in use by another process`);
+      console.log(`    ${dim("  Try: npx crawlio-browser --portal --port 3002")}`);
+      return;
+    }
+    console.log(`    ${yellow("!")} launchd loaded but server not responding, falling back...`);
+    return startDetachedServer(serverPath, nodePath);
+  }
+  return startDetachedServer(serverPath, nodePath);
+}
+async function startDetachedServer(serverPath, nodePath) {
+  const crawlioDir = join(HOME, ".crawlio");
+  mkdirSync(crawlioDir, { recursive: true });
+  const pidFile = join(crawlioDir, "server.pid");
+  const child = spawn(nodePath, [serverPath, "--portal"], {
+    detached: true,
+    stdio: "ignore"
+  });
+  child.unref();
+  if (child.pid) {
+    writeFileSync(pidFile, String(child.pid));
+  }
+  const result = await waitForHealth(5, 1e3);
+  if (result.ok) {
+    console.log(`    ${green("+")} Server running on ${PORTAL_URL}`);
+    console.log(`    ${dim("  PID saved to ~/.crawlio/server.pid")}`);
+  } else {
+    const portFree = await isPortFree(3001);
+    if (!portFree) {
+      console.log(`    ${yellow("!")} Port 3001 is already in use by another process`);
+      console.log(`    ${dim("  Try: npx crawlio-browser --portal --port 3002")}`);
+    } else {
+      console.log(`    ${yellow("!")} Server started but health check failed \u2014 check logs`);
+    }
+  }
+}
+function installBrowserSkill(dryRun) {
+  console.log("");
+  console.log(`  Installing browser-automation skill...`);
+  const claudeDir = join(HOME, ".claude");
+  if (!existsSync(claudeDir)) {
+    console.log(`    ${dim("  i ~/.claude not found \u2014 skipping skill install")}`);
+    return;
+  }
+  const destDir = join(claudeDir, "skills", "browser-automation");
+  const moduleDir = dirname(fileURLToPath(import.meta.url));
+  const skillSrcDir = resolve(moduleDir, "..", "..", "skills", "browser-automation");
+  if (!existsSync(join(skillSrcDir, "SKILL.md"))) {
+    console.log(`    ${yellow("!")} Skill source not found at ${dim(skillSrcDir)}`);
+    return;
+  }
+  if (dryRun) {
+    console.log(`    ${dim("[dry-run]")} Would copy SKILL.md + reference.md to ${destDir}`);
+    return;
+  }
+  mkdirSync(destDir, { recursive: true });
+  for (const file of ["SKILL.md", "reference.md"]) {
+    const src = join(skillSrcDir, file);
+    if (existsSync(src)) {
+      copyFileSync(src, join(destDir, file));
+    }
+  }
+  console.log(`    ${green("+")} Browser automation skill installed to ${dim(destDir)}`);
+}
+function installPlugin(dryRun) {
+  console.log("");
+  console.log(`  Installing crawlio-plugin skills...`);
+  const pluginsDir = join(HOME, ".crawlio", "plugins");
+  const pluginDir = join(pluginsDir, "crawlio-plugin");
+  if (dryRun) {
+    console.log(`    ${dim("[dry-run]")} Plugin target: ${pluginDir}`);
+    console.log(`    ${dim("[dry-run]")} Would clone: https://github.com/Crawlio-app/crawlio-plugin.git`);
+    return;
+  }
+  mkdirSync(pluginsDir, { recursive: true });
+  if (existsSync(join(pluginDir, "package.json"))) {
+    console.log(`    ${green("+")} Plugin already installed at ${dim(pluginDir)}`);
+    try {
+      execFileSync("git", ["-C", pluginDir, "pull", "--ff-only"], { stdio: "ignore", timeout: 15e3 });
+      console.log(`    ${green("+")} Updated to latest`);
+    } catch {
+    }
+  } else {
+    try {
+      execFileSync("git", ["clone", "https://github.com/Crawlio-app/crawlio-plugin.git", pluginDir], {
+        timeout: 3e4,
+        stdio: "pipe"
+      });
+      console.log(`    ${green("+")} Cloned to ${dim(pluginDir)}`);
+    } catch (error) {
+      const msg = error instanceof Error ? error.message : String(error);
+      console.log(`    ${yellow("!")} Clone failed: ${msg.slice(0, 150)}`);
+      return;
+    }
+  }
+  const claudeSkillsDir = join(HOME, ".claude", "skills");
+  if (existsSync(join(HOME, ".claude"))) {
+    mkdirSync(claudeSkillsDir, { recursive: true });
+    try {
+      const allFiles = readdirSync(pluginDir, { recursive: true, encoding: "utf-8" });
+      const skillFiles = allFiles.filter(
+        (f) => f.includes(`skills${sep}`) && f.endsWith(".md")
+      ).map((f) => join(pluginDir, f));
+      for (const skillFile of skillFiles) {
+        const name = basename(skillFile);
+        const dest = join(claudeSkillsDir, `crawlio-${name}`);
+        writeFileSync(dest, readFileSync(skillFile, "utf-8"));
+      }
+      if (skillFiles.length > 0) {
+        console.log(`    ${green("+")} ${skillFiles.length} skills copied to ${dim(claudeSkillsDir)}`);
+      }
+    } catch {
+    }
+  }
+}
+async function portalFlow(options) {
+  await ensurePortalRunning(options.dryRun);
+  console.log("");
+  console.log(`  Configuring MCP clients (portal mode)...`);
+  runAddMcp(options);
+}
+async function configureMetaMcp(found, options) {
+  console.log(`  Found MCP config at ${cyan(found.path)}`);
+  if (isAlreadyConfigured(found.config)) {
+    console.log(`    ${green("+")} crawlio-browser already configured \u2014 skipping`);
+    return;
+  }
+  if (!options.yes) {
+    const proceed = await confirm("Add crawlio-browser to this config?");
+    if (!proceed) {
+      console.log(`    ${dim("Skipped")}`);
+      return;
+    }
+  }
+  const entry = options.portal ? buildPortalEntry() : buildStdioEntry({ full: options.full });
+  if (options.dryRun) {
+    console.log(`    ${dim("[dry-run]")} Would add to ${found.path}:`);
+    console.log(`    ${dim("[dry-run]")} "crawlio-browser": ${JSON.stringify(entry)}`);
+    return;
+  }
+  found.config.mcpServers["crawlio-browser"] = entry;
+  writeFileSync(found.path, JSON.stringify(found.config, null, 2) + "\n");
+  console.log(`    ${green("+")} Added crawlio-browser to ${found.path}`);
+}
+function configureStdioClients(options) {
+  console.log("");
+  console.log(`  Configuring MCP clients (stdio mode)...`);
+  runAddMcp(options);
+}
+function runAddMcp(options) {
+  const npxBin = platform() === "win32" ? "npx.cmd" : "npx";
+  const args = buildAddMcpArgs(options);
+  if (options.dryRun) {
+    console.log(`    ${dim("[dry-run]")} Would run: ${npxBin} ${args.join(" ")}`);
+    return;
+  }
+  try {
+    const output = execFileSync(npxBin, args, {
+      encoding: "utf-8",
+      timeout: 6e4,
+      env: { ...process.env, npm_config_yes: "true" }
+    });
+    const lines = output.split("\n").filter((l) => l.trim());
+    let configuredCount = 0;
+    for (const line of lines) {
+      const trimmed = line.trim();
+      if (trimmed) {
+        console.log(`    ${green("+")} ${trimmed}`);
+        configuredCount++;
+      }
+    }
+    if (configuredCount === 0) {
+      console.log(`    ${dim("  add-mcp ran but no clients detected")}`);
+    }
+  } catch (error) {
+    const errObj = error;
+    const code = errObj?.code;
+    const msg = error instanceof Error ? error.message : String(error);
+    if (code === "ENOENT") {
+      console.log(`    ${yellow("!")} ${npxBin} not found in PATH \u2014 install Node.js 18+ and retry`);
+    } else if (code === "ETIMEDOUT" || msg.includes("ETIMEDOUT") || msg.includes("timed out")) {
+      console.log(`    ${yellow("!")} add-mcp timed out \u2014 check network and retry`);
+    } else {
+      console.log(`    ${yellow("!")} add-mcp failed: ${msg.slice(0, 200)}`);
+    }
+    const target = options.portal ? MCP_URL : "crawlio-browser";
+    console.log(`    ${dim(`  Manual: npx add-mcp ${target} --name crawlio-browser --global`)}`);
+  }
+}
+function printBanner() {
+  console.log("");
+  console.log(`  ${bold("Crawlio Browser Init")}`);
+  console.log(`  ${dim("Browser automation via MCP")}`);
+  console.log("");
+}
+function printExtensionLink() {
+  console.log("");
+  console.log(`  ${bold("Chrome Extension:")}`);
+  console.log(`    Install from ${cyan("https://crawlio.app/agent")}`);
+}
+async function printSummary(options) {
+  console.log("");
+  console.log(`  ${bold("Status:")}`);
+  if (options.portal) {
+    const health = await healthCheck();
+    if (health.ok) {
+      console.log(`    ${green("+")} Portal running on ${cyan(PORTAL_URL)} (${health.toolCount ?? "?"} tools)`);
+    } else {
+      console.log(`    ${yellow("!")} Portal not responding \u2014 start with: npx crawlio-browser --portal`);
+    }
+    if (health.bridgeConnected) {
+      console.log(`    ${green("+")} Chrome extension connected`);
+    } else {
+      console.log(`    ${dim("  i Chrome extension not connected yet")}`);
+    }
+  } else {
+    console.log(`    ${green("+")} Code mode configured (3 tools: search, execute, connect_tab)`);
+    console.log(`    ${dim("  i 125 commands searchable via the search tool")}`);
+  }
+  if (options.plugin) {
+    const pluginDir = join(HOME, ".crawlio", "plugins", "crawlio-plugin");
+    if (existsSync(pluginDir)) {
+      console.log(`    ${green("+")} Plugin installed`);
+    } else {
+      console.log(`    ${yellow("!")} Plugin not installed`);
+    }
+  } else {
+    console.log(`    ${dim("  i Plugin skipped (use --plugin to install)")}`);
+  }
+  console.log("");
+  console.log(`  ${bold("Next steps:")}`);
+  console.log(`    1. Install the Chrome extension ${dim("-> https://crawlio.app/agent")}`);
+  console.log(`    2. Open any configured MCP client and use crawlio-browser tools`);
+  if (!options.portal) {
+    console.log(`    ${dim("  Tip: use --portal for multi-client sharing or ChatGPT Desktop")}`);
+  }
+  console.log("");
+}
+async function runInit(argv) {
+  const options = parseFlags(argv);
+  printBanner();
+  if (options.dryRun) {
+    console.log(`  ${bold(yellow("[DRY RUN]"))} \u2014 showing what init would do without executing
+`);
+  }
+  const nodeVersion = parseInt(process.versions.node.split(".")[0], 10);
+  if (nodeVersion < 18) {
+    console.log(`  ${yellow("!")} Node.js ${process.versions.node} detected \u2014 18+ required`);
+    console.log(`    ${dim("Install via: https://nodejs.org")}`);
+    process.exit(1);
+  }
+  if (options.portal) {
+    await portalFlow(options);
+  } else {
+    const mcpConfig = findMcpConfig();
+    if (mcpConfig) {
+      await configureMetaMcp(mcpConfig, options);
+    } else {
+      configureStdioClients(options);
+    }
+  }
+  installBrowserSkill(options.dryRun);
+  if (options.plugin) {
+    installPlugin(options.dryRun);
+  }
+  if (!options.dryRun) {
+    printExtensionLink();
+    await printSummary(options);
+  }
+}
+export {
+  buildAddMcpArgs,
+  buildPortalEntry,
+  buildStdioEntry,
+  findMcpConfig,
+  installBrowserSkill,
+  isAlreadyConfigured,
+  parseFlags,
+  runInit
+};

package/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "crawlio-browser",
+  "version": "1.3.0",
+  "description": "MCP server with 87 CDP-backed tools for browser automation — screenshots, DOM, network capture, framework detection, cookies, storage, performance metrics via Chrome",
+  "type": "module",
+  "main": "dist/mcp-server/index.js",
+  "bin": {
+    "crawlio-browser": "bin/crawlio-browser.js"
+  },
+  "files": [
+    "bin/crawlio-browser.js",
+    "dist/mcp-server/",
+    "skills/",
+    ".claude-plugin/",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "npm run build:server && npm run build:extension",
+    "build:server": "node -e \"require('fs').rmSync('dist/mcp-server',{recursive:true,force:true})\" && tsup src/mcp-server/index.ts --format esm --dts --outDir dist/mcp-server",
+    "prepublishOnly": "npm run typecheck && npm run test && npm run build:server",
+    "build:extension": "rm -rf dist/extension && tsup src/extension/background.ts src/extension/popup.ts src/extension/welcome.ts --format iife --outDir dist/extension --define.__DEV__=false --minify && cd dist/extension && mv background.global.js background.js && mv popup.global.js popup.js && mv welcome.global.js welcome.js && cd ../.. && cp src/extension/manifest.prod.json dist/extension/manifest.json && cp src/extension/popup.html src/extension/popup.css src/extension/welcome.html src/extension/welcome.css src/extension/icon16.png src/extension/icon32.png src/extension/icon48.png src/extension/icon128.png dist/extension/",
+    "build:dev": "rm -rf dist/extension-dev && tsup src/extension/background.ts src/extension/popup.ts src/extension/welcome.ts --format iife --outDir dist/extension-dev --define.__DEV__=true && cd dist/extension-dev && mv background.global.js background.js && mv popup.global.js popup.js && mv welcome.global.js welcome.js && cd ../.. && cp src/extension/manifest.dev.json dist/extension-dev/manifest.json && cp src/extension/popup.html src/extension/popup.css src/extension/welcome.html src/extension/welcome.css src/extension/icon*.png dist/extension-dev/",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
+    "typecheck": "tsc --noEmit && tsc --noEmit -p tsconfig.extension.json",
+    "typecheck:extension": "tsc --noEmit -p tsconfig.extension.json",
+    "dev": "tsup src/mcp-server/index.ts --format esm --watch",
+    "setup": "node dist/mcp-server/index.js init"
+  },
+  "keywords": [
+    "mcp",
+    "crawlio",
+    "browser-automation",
+    "chrome",
+    "model-context-protocol",
+    "ai",
+    "screenshots",
+    "dom",
+    "web-scraping"
+  ],
+  "homepage": "https://crawlio.app/agent",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Crawlio-app/crawlio-browser-mcp"
+  },
+  "license": "MIT",
+  "engines": {
+    "node": ">=18"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.8.0",
+    "ws": "^8.18.1",
+    "zod": "^3.24.2"
+  },
+  "devDependencies": {
+    "@types/chrome": "^0.0.287",
+    "@types/ws": "^8.18.0",
+    "@vitest/coverage-v8": "^4.0.18",
+    "sharp": "^0.34.5",
+    "tsup": "^8.4.0",
+    "typescript": "^5.6.2",
+    "vitest": "^4.0.18"
+  }
+}