@iflow-mcp/jakeliume-webpeel 0.22.0

package/llms.txt

@@ -0,0 +1,105 @@
+# WebPeel — Web data for AI agents
+
+> Fetch any URL as clean markdown. Auto-handles JS rendering, anti-bot, 55+ domain extractors. One call.
+
+## Install & Quick Start
+
+npm install -g webpeel
+webpeel "https://example.com"             # Static pages — instant
+webpeel "https://polymarket.com" --render  # JS/React/SPA sites — uses browser
+webpeel "https://cloudflare-site.com" --stealth  # Anti-bot protected sites
+
+## IMPORTANT: When to use --render
+
+Most sites work without flags. Use --render (-r) for JavaScript-heavy sites:
+- React, Vue, Angular, Next.js, Svelte apps
+- SPAs: Polymarket, Airbnb, Booking.com, Zillow, Indeed, Expedia
+- Any page where content loads AFTER page render
+- If you get empty/sparse content, retry with --render
+
+Many popular SPAs auto-detect and render automatically (no flag needed):
+Twitter/X, Reddit, YouTube, Amazon, GitHub, Polymarket, and 50+ more.
+
+## CLI Commands
+
+webpeel <url>                    Fetch URL as clean markdown
+webpeel <url> --render           Fetch with browser rendering (JS sites)
+webpeel <url> --stealth          Anti-bot bypass (Cloudflare, Vercel)
+webpeel <url> --json             Full JSON output with metadata
+webpeel <url> --clean            AI-optimized (no URLs/noise)
+webpeel <url> -q "question"      Ask a question about the page (no LLM needed)
+webpeel <url> --scroll-extract   Scroll to load lazy/infinite content first
+webpeel search "query"           Web search via DuckDuckGo
+webpeel search "query" --site ebay   Site-specific search
+webpeel screenshot <url>         Take a screenshot
+webpeel screenshot <url> --full-page  Full-page screenshot
+webpeel crawl <url>              Crawl site pages
+webpeel map <url>                Discover site URLs
+webpeel batch <file>             Batch fetch from file
+webpeel mcp                      Start MCP server for AI tools
+
+## API
+
+POST https://api.webpeel.dev/v1/fetch
+Authorization: Bearer wp_your_key_here
+Content-Type: application/json
+
+{"url": "https://example.com", "format": "markdown"}
+
+# With browser rendering:
+{"url": "https://polymarket.com", "format": "markdown", "render": true}
+
+# With stealth mode:
+{"url": "https://protected-site.com", "format": "markdown", "stealth": true}
+
+## Domain Extractors (no browser needed — instant)
+
+55+ sites return structured data automatically via their APIs:
+Amazon, Reddit, GitHub, Twitter/X, YouTube, Wikipedia, HN, ArXiv,
+StackOverflow, NPM, PyPI, Spotify, IMDb, ESPN, eBay, Zillow,
+Polymarket, Kalshi, LinkedIn, Medium, Substack, and more.
+
+Example: webpeel "https://github.com/webpeel/webpeel" returns structured
+repo data (stars, description, languages) — no scraping needed.
+
+## MCP Integration (for AI agents)
+
+Add to your AI tool config (Claude, Cursor, Windsurf, etc.):
+{
+  "mcpServers": {
+    "webpeel": {
+      "command": "npx",
+      "args": ["webpeel", "mcp"]
+    }
+  }
+}
+
+Provides tools: webpeel (smart router), webpeel_read, webpeel_search,
+webpeel_screenshot, webpeel_crawl, webpeel_map, webpeel_extract, webpeel_monitor.
+
+## Programmatic (Node.js)
+
+import { peel } from 'webpeel';
+
+// Static page
+const result = await peel('https://example.com');
+console.log(result.content);  // Clean markdown
+
+// JS-rendered SPA
+const spa = await peel('https://polymarket.com/@user', { render: true });
+
+// With auto-scroll for infinite content
+const full = await peel('https://imgur.com/gallery', {
+  render: true,
+  autoScroll: true,
+});
+
+## Troubleshooting
+
+| Problem | Solution |
+|---------|----------|
+| Empty/sparse content | Add --render (site needs JavaScript) |
+| Blocked by Cloudflare | Add --stealth |
+| Only partial content | Add --scroll-extract (loads lazy content) |
+| Need all data from SPA | Check if domain extractor exists (instant API data) |
+| Rate limited (429) | Wait and retry, or use --stealth |

package/package.json

@@ -0,0 +1,189 @@
+{
+  "name": "@iflow-mcp/jakeliume-webpeel",
+  "version": "0.22.0",
+  "description": "The web data layer for AI agents — fetch, search, crawl, extract, screenshot, and monitor the web with 55+ domain extractors and MCP.",
+  "author": "Jake Liu",
+  "license": "SEE LICENSE IN LICENSE",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "bin": {
+    "iflow-mcp-jakeliume-webpeel": "dist/mcp/server.js"
+  },
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./integrations/langchain": {
+      "import": "./dist/integrations/langchain.js",
+      "types": "./dist/integrations/langchain.d.ts"
+    },
+    "./integrations/llamaindex": {
+      "import": "./dist/integrations/llamaindex.js",
+      "types": "./dist/integrations/llamaindex.d.ts"
+    },
+    "./integrations": {
+      "import": "./dist/integrations/index.js",
+      "types": "./dist/integrations/index.d.ts"
+    }
+  },
+  "files": [
+    "dist/index.js",
+    "dist/index.d.ts",
+    "dist/cli.js",
+    "dist/cli.bundle.cjs",
+    "dist/cli.d.ts",
+    "dist/types.js",
+    "dist/types.d.ts",
+    "dist/core",
+    "dist/ee",
+    "dist/integrations",
+    "dist/mcp",
+    "dist/server",
+    "dist/cli/",
+    "README.md",
+    "LICENSE",
+    "llms.txt",
+    "dist/cli-auth.js",
+    "dist/cli-auth.d.ts",
+    "dist/cache.js",
+    "dist/cache.d.ts"
+  ],
+  "scripts": {
+    "build": "tsc && cp src/server/openapi.yaml dist/server/openapi.yaml",
+    "bundle": "node scripts/bundle-cli.js",
+    "dev": "tsc --watch",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "lint": "eslint src/ --ext .ts --max-warnings 0",
+    "lint:fix": "eslint src/ --ext .ts --fix",
+    "format": "prettier --write 'src/**/*.ts'",
+    "format:check": "prettier --check 'src/**/*.ts'",
+    "typecheck": "tsc --noEmit",
+    "prepublishOnly": "bash scripts/pre-publish.sh",
+    "serve": "node dist/server/app.js",
+    "mcp": "node dist/mcp/server.js",
+    "preversion": "npm run build && npm test && bash scripts/pre-publish-gate.sh",
+    "version": "bash scripts/postversion.sh",
+    "eval": "npx tsx scripts/eval-smart-search.ts --production",
+    "eval:fetch": "npx tsx scripts/eval-fetch.ts --production",
+    "eval:local": "npx tsx scripts/eval-smart-search.ts --local",
+    "eval:deploy": "bash scripts/workflow/eval-on-deploy.sh",
+    "verify:verdict-ui": "npx tsx scripts/verify-transit-verdict-ui.ts --screenshot"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/webpeel/webpeel.git"
+  },
+  "bugs": {
+    "url": "https://github.com/webpeel/webpeel/issues"
+  },
+  "homepage": "https://webpeel.dev",
+  "keywords": [
+    "web-scraper",
+    "ai-agent",
+    "mcp-server",
+    "mcp",
+    "model-context-protocol",
+    "playwright",
+    "markdown",
+    "fetcher",
+    "web-fetcher",
+    "claude",
+    "claude-code",
+    "cursor",
+    "vscode",
+    "windsurf",
+    "cline",
+    "gemini",
+    "gpt",
+    "llm",
+    "ai",
+    "web-to-markdown",
+    "headless-browser",
+    "web-scraping",
+    "firecrawl-alternative",
+    "crawl4ai-alternative",
+    "web-data",
+    "token-efficiency",
+    "rag",
+    "web-search",
+    "screenshot",
+    "content-extraction",
+    "web-crawler",
+    "site-crawler",
+    "pdf-extraction",
+    "youtube-transcript",
+    "cloudflare-bypass",
+    "stealth-browser",
+    "structured-data"
+  ],
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.0.4",
+    "bull": "^4.16.5",
+    "cheerio": "^1.0.0",
+    "cloakbrowser": "^0.1.8",
+    "commander": "^12.0.0",
+    "cycletls": "^2.0.5",
+    "helmet": "^8.1.0",
+    "ioredis": "^5.10.0",
+    "lru-cache": "^11.0.2",
+    "mammoth": "^1.11.0",
+    "nodemailer": "^8.0.1",
+    "ora": "^8.0.1",
+    "pdf-parse": "^1.1.4",
+    "pixelmatch": "^7.1.0",
+    "playwright": "npm:rebrowser-playwright@^1.52.0",
+    "playwright-extra": "^4.3.6",
+    "pngjs": "^7.0.0",
+    "prom-client": "^15.1.3",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
+    "rate-limiter-flexible": "^10.0.1",
+    "resend": "^6.9.3",
+    "tesseract.js": "^7.0.0",
+    "turndown": "^7.2.0",
+    "turndown-plugin-gfm": "^1.0.2",
+    "undici": "^7.2.0",
+    "update-notifier": "^7.3.1",
+    "youtube-transcript": "^1.2.1",
+    "youtube-transcript-plus": "^1.2.0"
+  },
+  "optionalDependencies": {
+    "@sentry/node": "^7.120.4",
+    "bcrypt": "^6.0.0",
+    "cors": "^2.8.5",
+    "express": "^4.21.2",
+    "jsonwebtoken": "^9.0.3",
+    "pg": "^8.18.0",
+    "stripe": "^20.3.1"
+  },
+  "devDependencies": {
+    "@types/bcrypt": "^6.0.0",
+    "@types/cors": "^2.8.17",
+    "@types/express": "^5.0.0",
+    "@types/jsonwebtoken": "^9.0.10",
+    "@types/node": "^22.0.0",
+    "@types/nodemailer": "^7.0.11",
+    "@types/pdf-parse": "^1.1.5",
+    "@types/pg": "^8.16.0",
+    "@types/pngjs": "^6.0.5",
+    "@types/supertest": "^6.0.3",
+    "@types/turndown": "^5.0.5",
+    "@typescript-eslint/eslint-plugin": "^8.57.1",
+    "@typescript-eslint/parser": "^8.57.1",
+    "esbuild": "^0.27.3",
+    "eslint": "^10.1.0",
+    "eslint-config-prettier": "^10.1.8",
+    "pdf-lib": "^1.17.1",
+    "prettier": "^3.8.1",
+    "puppeteer": "^24.37.5",
+    "supertest": "^7.2.2",
+    "typescript": "^5.6.0",
+    "vitest": "^4.1.2"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "mcpName": "io.github.webpeel/webpeel"
+}