@robot-resources/scraper 0.1.1 → 0.1.2

package/bin/setup.js

@@ -1,15 +1,36 @@
 #!/usr/bin/env node
 
 /**
- * robot-resources-scraper — Setup wizard for @robot-resources/scraper.
+ * robot-resources-scraper — CLI for @robot-resources/scraper.
  *
- * Triggered via `npx @robot-resources/scraper`.
- * Offers optional GitHub login and shows usage instructions.
+ * Usage:
+ *   npx @robot-resources/scraper <url>              → compressed markdown to stdout
+ *   npx @robot-resources/scraper <url> --json       → full result as JSON
+ *   npx @robot-resources/scraper <url> --mode fast  → specific fetch mode
+ *   npx @robot-resources/scraper                    → show usage & setup info
  */
 
-import { readFileSync, writeFileSync, copyFileSync, mkdirSync, existsSync } from "node:fs";
+import { readFileSync } from "node:fs";
 import { homedir } from "node:os";
-import { join } from "node:path";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+// ─── Arg parsing (zero deps) ────────────────────────────────────────────────
+
+const args = process.argv.slice(2);
+let url = null;
+let mode = "auto";
+let timeout = undefined;
+let json = false;
+let help = false;
+
+for (let i = 0; i < args.length; i++) {
+  if (args[i] === "--mode" && args[i + 1]) { mode = args[++i]; continue; }
+  if (args[i] === "--timeout" && args[i + 1]) { timeout = Number(args[++i]); continue; }
+  if (args[i] === "--json") { json = true; continue; }
+  if (args[i] === "--help" || args[i] === "-h") { help = true; continue; }
+  if (!url && /^https?:\/\//.test(args[i])) { url = args[i]; continue; }
+}
 
 // ─── ANSI helpers ────────────────────────────────────────────────────────────
 
@@ -23,113 +44,82 @@ const c = {
   blue: "\x1b[34m",
 };
 
-function success(msg) { console.log(`  ${c.green}✓${c.reset} ${msg}`); }
-function step(msg) { console.log(`  ${c.cyan}→${c.reset} ${msg}`); }
-function info(msg) { console.log(`  ${c.dim}${msg}${c.reset}`); }
-function warn(msg) { console.log(`  ${c.yellow}!${c.reset} ${msg}`); }
-
-// ─── Config helpers (inline — no external deps for this bin) ─────────────────
+// ─── Config helpers ──────────────────────────────────────────────────────────
 
-const CONFIG_DIR = join(homedir(), ".robot-resources");
-const CONFIG_FILE = join(CONFIG_DIR, "config.json");
+const CONFIG_FILE = join(homedir(), ".robot-resources", "config.json");
 
 function readConfig() {
   try { return JSON.parse(readFileSync(CONFIG_FILE, "utf-8")); }
   catch { return {}; }
 }
 
-// ─── MCP auto-config ─────────────────────────────────────────────────────────
-
-const MCP_KEY = "robot-resources-scraper";
-const MCP_ENTRY = { command: "npx", args: ["-y", "@robot-resources/scraper-mcp"] };
-
-function detectAgents() {
-  const home = homedir();
-  const agents = [
-    {
-      name: "Claude Desktop",
-      configPath: process.platform === "darwin"
-        ? join(home, "Library", "Application Support", "Claude", "claude_desktop_config.json")
-        : join(home, ".config", "Claude", "claude_desktop_config.json"),
-    },
-    { name: "Cursor", configPath: join(home, ".cursor", "mcp.json") },
-  ];
-  return agents.filter((a) => existsSync(a.configPath) || existsSync(join(a.configPath, "..")));
-}
+// ─── Scrape mode: URL provided ──────────────────────────────────────────────
 
-function configureAgentMCP() {
-  const agents = detectAgents();
-  const results = [];
-
-  for (const agent of agents) {
-    try {
-      let config;
-      try { config = JSON.parse(readFileSync(agent.configPath, "utf-8")); }
-      catch { config = {}; }
-
-      config.mcpServers = config.mcpServers || {};
-      if (config.mcpServers[MCP_KEY]) {
-        results.push({ name: agent.name, action: "exists" });
-        continue;
-      }
-
-      // Backup before modifying
-      if (existsSync(agent.configPath)) {
-        copyFileSync(agent.configPath, `${agent.configPath}.bak`);
-      }
-
-      config.mcpServers[MCP_KEY] = MCP_ENTRY;
-      mkdirSync(join(agent.configPath, ".."), { recursive: true });
-      writeFileSync(agent.configPath, JSON.stringify(config, null, 2) + "\n");
-      results.push({ name: agent.name, action: "added" });
-    } catch (err) {
-      results.push({ name: agent.name, action: "error", reason: err.message });
-    }
+async function runScrape(targetUrl, opts) {
+  const { scrape } = await import("@robot-resources/scraper");
+  const result = await scrape(targetUrl, {
+    mode: opts.mode,
+    timeout: opts.timeout,
+  });
+
+  if (opts.json) {
+    process.stdout.write(JSON.stringify(result) + "\n");
+  } else {
+    process.stdout.write(result.markdown + "\n");
   }
-  return results;
 }
 
-// ─── Main ────────────────────────────────────────────────────────────────────
+// ─── Usage mode: no URL ─────────────────────────────────────────────────────
+
+function showUsage() {
+  const __dirname = dirname(fileURLToPath(import.meta.url));
+  const pkg = JSON.parse(readFileSync(join(__dirname, "..", "package.json"), "utf-8"));
 
-async function main() {
-  console.log(`\n  ${c.blue}${c.bold}██ Robot Resources — Scraper Setup${c.reset}\n`);
+  console.log(`\n  ${c.blue}${c.bold}██ Robot Resources — Scraper v${pkg.version}${c.reset}\n`);
 
-  // Step 1: Auth status
+  // Auth status
   const config = readConfig();
   if (config.api_key) {
-    success(`Logged in as ${config.user_name || config.user_email || "unknown"}`);
+    console.log(`  ${c.green}✓${c.reset} Logged in as ${config.user_name || config.user_email || "unknown"}`);
   } else {
-    info("Not logged in. Scraper works without login.");
-    info(`To enable telemetry, run: ${c.cyan}npx robot-resources${c.reset}`);
+    console.log(`  ${c.dim}Not logged in. Scraper works without login.${c.reset}`);
+    console.log(`  ${c.dim}To enable telemetry, run: ${c.cyan}npx robot-resources${c.reset}`);
   }
 
-  // Step 2: MCP auto-config
-  console.log("");
-  step("Configuring MCP in detected agents...");
+  // CLI usage
+  console.log(`\n  ${c.blue}${c.bold}── Command Line ──${c.reset}\n`);
+  console.log(`  ${c.cyan}npx @robot-resources/scraper ${c.dim}<url>${c.reset}                  Compressed markdown`);
+  console.log(`  ${c.cyan}npx @robot-resources/scraper ${c.dim}<url>${c.cyan} --json${c.reset}            Full result as JSON`);
+  console.log(`  ${c.cyan}npx @robot-resources/scraper ${c.dim}<url>${c.cyan} --mode stealth${c.reset}    Stealth fetch mode`);
+  console.log(`  ${c.cyan}npx @robot-resources/scraper ${c.dim}<url>${c.cyan} --timeout 15000${c.reset}   Custom timeout (ms)`);
 
-  const mcpResults = configureAgentMCP();
-  if (mcpResults.length === 0) {
-    info("No supported agents detected (Claude Desktop, Cursor)");
-  } else {
-    for (const r of mcpResults) {
-      if (r.action === "added") success(`${r.name}: scraper MCP configured`);
-      else if (r.action === "exists") success(`${r.name}: already configured`);
-      else warn(`${r.name}: ${r.reason || r.action}`);
-    }
-  }
-
-  // Step 3: Usage
-  console.log(`\n  ${c.blue}${c.bold}── Ready ──${c.reset}\n`);
-  console.log("  Use as a library:");
+  // Library usage
+  console.log(`\n  ${c.blue}${c.bold}── As a Library ──${c.reset}\n`);
   console.log(`  ${c.dim}import { scrape } from '@robot-resources/scraper';${c.reset}`);
   console.log(`  ${c.dim}const result = await scrape('https://example.com');${c.reset}`);
-  console.log("");
-  console.log("  Use via MCP (already configured above):");
-  console.log(`  ${c.dim}Your agent can call scraper_compress_url(url)${c.reset}`);
+
+  // MCP usage (generic, not agent-specific)
+  console.log(`\n  ${c.blue}${c.bold}── Via MCP ──${c.reset}\n`);
+  console.log(`  ${c.dim}Add to your agent's MCP config:${c.reset}`);
+  console.log(`  ${c.dim}{${c.reset}`);
+  console.log(`  ${c.dim}  "mcpServers": {${c.reset}`);
+  console.log(`  ${c.dim}    "scraper": {${c.reset}`);
+  console.log(`  ${c.dim}      "command": "npx",${c.reset}`);
+  console.log(`  ${c.dim}      "args": ["-y", "@robot-resources/scraper-mcp"]${c.reset}`);
+  console.log(`  ${c.dim}    }${c.reset}`);
+  console.log(`  ${c.dim}  }${c.reset}`);
+  console.log(`  ${c.dim}}${c.reset}`);
+
   console.log("");
 }
 
-main().catch((err) => {
-  console.error(`\n  Setup failed: ${err.message}\n`);
-  process.exit(1);
-});
+// ─── Main ────────────────────────────────────────────────────────────────────
+
+if (url && !help) {
+  runScrape(url, { mode, timeout, json }).catch((err) => {
+    process.stderr.write(`Error: ${err.message}\n`);
+    process.exit(1);
+  });
+} else {
+  showUsage();
+}

package/package.json

@@ -1,9 +1,8 @@
 {
   "name": "@robot-resources/scraper",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "Context compression for AI agents. Fetch -> Extract -> Convert pipeline without LLM dependency.",
   "author": "Robot Resources",
-  "publishConfig": { "access": "public" },
   "license": "MIT",
   "type": "module",
   "bin": {