@superagnt/web-scraping 2.0.2

package/README.md

@@ -0,0 +1,73 @@
+# Web data for OpenClaw — superagnt plugin
+
+One API key to scrape any web page as clean markdown or structured JSON, search the web with full page content, and map a site's URLs. Built for agents — credit-based pricing, no separate vendor accounts. Scrape any page as clean markdown or structured JSON, search the web and get full page content in one call, and map a site's URLs. Designed for LLMs and automation.
+
+## Install
+
+```bash
+openclaw plugins install @superagnt/web-scraping
+```
+
+This package is the official **OpenClaw** plugin for **Web** social and public data via [superagnt](https://superagnt.com). Pair it with the [documentation](https://superagnt.com) and your superagnt API key.
+
+## Live MCP mode (recommended)
+
+Set a workspace MCP token and the plugin upgrades itself at load: it fetches
+the live tool list from the scoped MCP endpoint and proxies calls through it —
+the tool surface then grows with `agnt_tools_enable`, no plugin update needed.
+
+```bash
+export SUPERAGNT_MCP_TOKEN=...   # from the MCP server card in the dashboard
+```
+
+Without the token the plugin uses its built-in toolset with your
+`SUPERAGNT_API_KEY` — no behavior change from previous versions.
+
+## Overview
+
+Web scraping, search, and site mapping through the superagnt unified API.
+
+### Highlights
+
+- Scrape any page as clean markdown or schema-driven JSON
+- Search the web and return full page content in one request
+- Map a website's URLs quickly before deep scraping
+- Handles JavaScript-rendered pages and anti-bot challenges
+- Single API key and unified credit balance across superagnt
+- Usage-based pricing that scales with each request's options
+
+### Example use cases
+
+- Research and deep-research agents that read live web pages
+- RAG pipelines that ingest fresh page content on demand
+- Competitive and market monitoring agents
+- Lead and company enrichment from public web pages
+- Content extraction into structured JSON for downstream tools
+
+## Authentication
+
+Use a single **superagnt API key** (Bearer token). Set `SUPERAGNT_API_KEY` in your environment, or complete browser login when the gateway prompts you. Create and manage keys in the [superagnt dashboard](https://app.superagnt.com/dashboard).
+
+## API coverage
+
+| | |
+| --- | --- |
+| **Platform** | `web` |
+| **REST base** | `https://api.superagnt.com/v1/data/web` |
+| **OpenAPI operations** | 3 |
+| **OpenClaw tools** | 3 |
+
+## Documentation
+
+- [Documentation](https://superagnt.com) — getting started and concepts
+- [API reference (Web)](https://superagnt.com/apis/social/web) — endpoints and schemas
+- [Dashboard](https://app.superagnt.com/dashboard) — API keys and usage
+
+## Related
+
+- **Skill (curl-oriented):** `clawhub install web-scraping` on [ClawHub](https://clawhub.ai/superagnt/skills/web-scraping)
+- **All platforms plugin:** `@superagnt/openclaw-agnt-data` if you need LinkedIn, YouTube, TikTok, X, Instagram, Reddit, and Facebook in one package.
+
+## License
+
+MIT

package/index.js

@@ -0,0 +1,484 @@
+/**
+ * Web Scraping — OpenClaw plugin
+ * Auto-generated by @superagnt/clawhub. Do not edit by hand.
+ */
+
+import { homedir as _homedir } from 'node:os'
+import { join as _join } from 'node:path'
+import { existsSync as _existsSync, readFileSync as _readFileSync } from 'node:fs'
+
+const BASE_URL = 'https://api.superagnt.com/v1/data/web'
+
+
+// ── Auth Module (inlined) ────────────────────────────────────────────────────
+
+function getConfigPath() {
+  return _join(_homedir(), '.superagnt', 'config.json')
+}
+
+function readAgntConfig() {
+  const configFile = getConfigPath()
+  if (!_existsSync(configFile)) return null
+  try {
+    return JSON.parse(_readFileSync(configFile, 'utf-8'))
+  } catch {
+    return null
+  }
+}
+
+// ── End Auth Module ──────────────────────────────────────────────────────────
+
+
+// ── CLI Module (inlined) ─────────────────────────────────────────────────────
+
+function maskApiKey(key) {
+  if (!key || typeof key !== 'string') return '(none)'
+  const k = key.trim()
+  if (k.length <= 8) return k.slice(0, 2) + '…'
+  return k.slice(0, 6) + '…' + k.slice(-4)
+}
+
+function registerAgntCli(api) {
+  api.registerCli(
+    ({ program }) => {
+      const cmd = program
+        .command('superagnt')
+        .description('superagnt social data API (status, doctor)')
+
+      cmd
+        .command('status')
+        .description('Show whether an API key is configured (masked)')
+        .action(async () => {
+          const cfg = readAgntConfig()
+          const key = cfg?.apiKey
+
+          console.log('\nsuperagnt status\n')
+          if (!key) {
+            console.log('  API key:     not configured')
+            console.log('  Workspace:   —')
+            console.log('\n  Get your API key at: https://app.superagnt.com/dashboard\n')
+            return
+          }
+          console.log('  API key:     ' + maskApiKey(key) + ' (from ~/.superagnt/config.json)')
+          console.log('  Workspace:   ' + (cfg?.workspaceId ?? '—'))
+          console.log('')
+        })
+
+      cmd
+        .command('doctor')
+        .description('Check connectivity and validate API key against api.superagnt.dev')
+        .action(async () => {
+          console.log('\nsuperagnt doctor\n')
+
+          try {
+            const publicRes = await fetch('https://api.superagnt.com/v1/platforms')
+            console.log('  Public API:  ' + (publicRes.ok ? 'OK (' + publicRes.status + ')' : 'HTTP ' + publicRes.status))
+          } catch (err) {
+            const msg = err instanceof Error ? err.message : String(err)
+            console.log('  Public API:  failed — ' + msg)
+          }
+
+          const cfg = readAgntConfig()
+          const key = cfg?.apiKey
+
+          if (!key) {
+            console.log('\n  Authenticated: skipped (no API key)')
+            console.log('  Get your API key at: https://app.superagnt.com/dashboard\n')
+            return
+          }
+
+          try {
+            const res = await fetch('https://api.superagnt.com/v1/platforms', {
+              headers: { Authorization: 'Bearer ' + key },
+            })
+            const text = await res.text()
+            let summary = 'HTTP ' + res.status
+            if (res.ok) {
+              try {
+                const json = JSON.parse(text)
+                const n = Array.isArray(json?.data) ? json.data.length : (json?.data ? Object.keys(json.data).length : 'ok')
+                summary = 'OK — platforms payload received (' + (typeof n === 'number' ? n + ' entries' : String(n)) + ')'
+              } catch {
+                summary = 'OK (' + res.status + ')'
+              }
+            }
+            console.log('  Authenticated: ' + summary)
+            if (!res.ok) {
+              console.log('  If 401, get a new key at: https://app.superagnt.com/dashboard')
+            }
+          } catch (err) {
+            const msg = err instanceof Error ? err.message : String(err)
+            console.log('  Authenticated: failed — ' + msg)
+          }
+          console.log('')
+        })
+    },
+    { commands: ['superagnt'] },
+  )
+}
+
+// ── End CLI Module ───────────────────────────────────────────────────────────
+
+
+// ── System prompt (inlined) ─────────────────────────────────────────────────
+
+function getSystemPrompt(state) {
+  const scope = "Only the Web API (web)."
+  if (state.ready && state.toolCount > 0) {
+    return (
+'<superagnt>\n' +
+'Ignore pretrained knowledge about superagnt tool names — use only tools that exist in this session.\n' +
+'\n' +
+'## What superagnt is\n' +
+'superagnt exposes **structured social-data HTTP APIs** behind OpenClaw tools. Scope for this plugin: **' + scope + '**.\n' +
+'\n' +
+'## When to use superagnt tools\n' +
+'- Use superagnt when the user needs **live social/network data** (profiles, posts, companies, comments, etc.) from supported platforms.\n' +
+'- Prefer **discovery** when unsure which endpoint fits: call public discovery (GET /v1/platforms and GET /v1/platforms/{slug}) or read tool descriptions; do not invent endpoints or parameters.\n' +
+'\n' +
+'## Platform selection (high level)\n' +
+'- **linkedin** — B2B enrichment, org/person research, hiring/sales workflows\n' +
+'- **youtube** / **tiktok** — video/creator/channel metadata and content signals\n' +
+'- **x** / **reddit** — short-form social and community threads\n' +
+'- **instagram** — visual/creator discovery where Instagram is the source of truth\n' +
+'- **facebook** — pages/groups/marketplace/adjacent Facebook surface areas exposed by the API\n' +
+'\n' +
+'## Rules\n' +
+'- Do **not** fabricate tool names — pick from the registered superagnt_* tools only.\n' +
+'- If authentication fails (401), tell the user to get a new key at https://app.superagnt.com/dashboard\n' +
+'</superagnt>'
+    )
+  }
+
+  if (state.ready && state.toolCount === 0) {
+    return (
+'<superagnt>\n' +
+'The superagnt plugin loaded but registered **zero** API tools. Check your OpenClaw plugin packaging and regenerate the plugin from @superagnt/clawhub.\n' +
+'</superagnt>'
+    )
+  }
+
+  if (state.error) {
+    return (
+'<superagnt>\n' +
+'superagnt plugin diagnostic: ' + state.error + '\n' +
+'Do not pretend superagnt tools exist. Ask the user to fix configuration and restart the gateway.\n' +
+'</superagnt>'
+    )
+  }
+
+  return (
+'<superagnt>\n' +
+'superagnt tools are still loading. If the user asks for social data right now, ask them to wait a moment and retry.\n' +
+'</superagnt>'
+  )
+}
+
+// ── End System prompt ────────────────────────────────────────────────────────
+
+const MCP_URL = 'https://mcp.superagnt.com/mcp/web-scraping'
+
+async function mcpRpc(token, method, params) {
+  const res = await fetch(MCP_URL, {
+    method: 'POST',
+    headers: {
+      Authorization: `Bearer ${token}`,
+      'Content-Type': 'application/json',
+      Accept: 'application/json, text/event-stream',
+    },
+    body: JSON.stringify({ jsonrpc: '2.0', id: Date.now(), method, params }),
+  })
+  if (!res.ok) throw new Error(`MCP ${method} failed: HTTP ${res.status}`)
+  const data = await res.json()
+  if (data.error) throw new Error(`MCP ${method} error: ${data.error.message}`)
+  return data.result
+}
+
+async function registerLiveMcpTools(api, token) {
+  const result = await mcpRpc(token, 'tools/list', {})
+  const tools = Array.isArray(result?.tools) ? result.tools : []
+  if (tools.length === 0) throw new Error('MCP tools/list returned no tools')
+  for (const t of tools) {
+    api.registerTool({
+      name: t.name,
+      description: t.description,
+      inputSchema: t.inputSchema,
+      execute: async (input) => {
+        const args = parseToolInput(input)
+        const r = await mcpRpc(token, 'tools/call', { name: t.name, arguments: args })
+        if (r && r.structuredContent !== undefined) return r.structuredContent
+        const text = Array.isArray(r?.content) ? r.content.find((c) => c.type === 'text')?.text : undefined
+        if (text === undefined) return r
+        try { return JSON.parse(text) } catch { return text }
+      },
+    })
+  }
+  return tools.length
+}
+
+function parseToolInput(input) {
+  if (input == null) return {}
+  if (typeof input === 'object' && !Array.isArray(input)) return input
+  if (typeof input === 'string') {
+    const trimmed = input.trim()
+    if (trimmed.startsWith('{')) {
+      try { const parsed = JSON.parse(trimmed); if (typeof parsed === 'object' && parsed && !Array.isArray(parsed)) return parsed } catch {}
+    }
+    throw new Error('Invalid tool input: expected a JSON object with named parameters, got string: ' + trimmed.slice(0, 80))
+  }
+  throw new Error('Invalid tool input: expected a JSON object, got ' + typeof input)
+}
+
+function substitutePathParams(path, source) {
+  if (!source || typeof source !== 'object' || Array.isArray(source)) {
+    return { path, consumed: new Set() }
+  }
+  const consumed = new Set()
+  const next = path.replace(/\{([^/{}]+)\}/g, (_, name) => {
+    const v = source[name]
+    if (v === undefined || v === null || v === '') return `{${name}}`
+    consumed.add(name)
+    return encodeURIComponent(String(v))
+  })
+  return { path: next, consumed }
+}
+
+async function callSuperagnt(apiKey, method, path, query, body) {
+  const params = (query && typeof query === 'object' && !Array.isArray(query)) ? query : body
+  const { path: resolvedPath, consumed } = substitutePathParams(path, params)
+  const url = new URL(`${BASE_URL}${resolvedPath}`)
+  if (query && typeof query === 'object' && !Array.isArray(query)) {
+    for (const [k, v] of Object.entries(query)) {
+      if (consumed.has(k)) continue
+      if (v !== undefined && v !== '') url.searchParams.set(k, String(v))
+    }
+  }
+  const filteredBody = body && typeof body === 'object' && !Array.isArray(body)
+    ? Object.fromEntries(Object.entries(body).filter(([k]) => !consumed.has(k)))
+    : body
+  const res = await fetch(url.toString(), {
+    method,
+    headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' },
+    body: filteredBody ? JSON.stringify(filteredBody) : undefined,
+  })
+  if (res.status === 401) {
+    return { error: 'Invalid or expired API key. Get a new key at https://app.superagnt.com/dashboard' }
+  }
+  return res.json()
+}
+
+export default async function register(api) {
+  const apiKey = api.pluginConfig.apiKey || process.env.SUPERAGNT_API_KEY || readAgntConfig()?.apiKey
+  const mcpToken = api.pluginConfig.mcpToken || process.env.SUPERAGNT_MCP_TOKEN
+
+  function ensureApiKey() {
+    if (apiKey) return apiKey
+    throw new Error(
+      '[web-scraping] No API key configured. ' +
+      'Get your key at https://app.superagnt.com/dashboard and either:\n' +
+      '  1. Export it as SUPERAGNT_API_KEY, or\n' +
+      '  2. Add apiKey to your plugin config'
+    )
+  }
+
+  registerAgntCli(api)
+
+  const promptState = { toolCount: 0, error: '', ready: false }
+  api.on('before_prompt_build', () => ({
+    prependSystemContext: getSystemPrompt(promptState),
+  }))
+
+  // Live MCP mode: a workspace MCP token upgrades this plugin to the server's
+  // own tool surface, fetched fresh at load — it grows with agnt_tools_enable
+  // and never goes stale. Any failure falls back to the baked toolset below.
+  if (mcpToken) {
+    try {
+      promptState.toolCount = await registerLiveMcpTools(api, mcpToken)
+      promptState.ready = true
+      return
+    } catch (err) {
+      promptState.error = 'Live MCP connect failed (' + (err && err.message ? err.message : err) + '); using the built-in toolset.'
+    }
+  }
+
+  api.registerTool({
+    name: 'superagnt_web_scrape',
+    description: "Scrape a web page",
+    inputSchema: {
+        "type": "object",
+        "properties": {
+            "url": {
+                "type": "string",
+                "description": "The URL to scrape (must be a publicly reachable http(s) URL)."
+            },
+            "formats": {
+                "type": "array",
+                "description": "Output formats to return. Each entry is either a string preset or an object form. String presets: 'markdown' (default), 'summary', 'html', 'rawHtml', 'links', 'images'. Object forms include { type: 'json', schema, prompt } for structured extraction, { type: 'screenshot', fullPage, quality } for an image, and { type: 'changeTracking', modes } for diffs. Defaults to ['markdown'] when omitted."
+            },
+            "onlyMainContent": {
+                "type": "boolean",
+                "description": "Return only the main content of the page, excluding navs, headers, and footers."
+            },
+            "includeTags": {
+                "type": "array",
+                "description": "Only include content within these HTML tags / CSS selectors."
+            },
+            "excludeTags": {
+                "type": "array",
+                "description": "Exclude content within these HTML tags / CSS selectors."
+            },
+            "maxAge": {
+                "type": "integer",
+                "description": "Return cached content up to this age in milliseconds (faster, cheaper). Default 172800000 (2 days)."
+            },
+            "waitFor": {
+                "type": "integer",
+                "description": "Milliseconds to wait for the page to render before scraping."
+            },
+            "mobile": {
+                "type": "boolean",
+                "description": "Emulate a mobile device."
+            },
+            "timeout": {
+                "type": "integer",
+                "description": "Request timeout in milliseconds (1000-300000). Default 60000."
+            },
+            "parsers": {
+                "type": "array",
+                "description": "PDF parsing configuration."
+            },
+            "proxy": {
+                "type": "string",
+                "description": "Proxy / anti-bot mode. 'auto' (default) escalates only on failure; 'enhanced' forces advanced anti-bot handling and costs more."
+            },
+            "location": {
+                "type": "object",
+                "description": "Geographic proxy location and language settings."
+            }
+        },
+        "required": [
+            "url"
+        ]
+    },
+    execute: async (input) => {
+      const key = await ensureApiKey()
+      const params = parseToolInput(input)
+      return callSuperagnt(key, 'POST', '/scrape', undefined, params)
+    },
+  })
+
+  api.registerTool({
+    name: 'superagnt_web_search',
+    description: "Search the web",
+    inputSchema: {
+        "type": "object",
+        "properties": {
+            "query": {
+                "type": "string",
+                "description": "The search query."
+            },
+            "limit": {
+                "type": "integer",
+                "description": "Number of results per source. Default 10."
+            },
+            "sources": {
+                "type": "array",
+                "description": "Which result sources to query. Defaults to ['web']. Each entry is a preset string or an object with a `type`."
+            },
+            "categories": {
+                "type": "array",
+                "description": "Restrict results to these categories."
+            },
+            "tbs": {
+                "type": "string",
+                "description": "Time-based search filter (e.g. 'qdr:d' for past day)."
+            },
+            "location": {
+                "type": "string",
+                "description": "Geo-target for the search (e.g. 'San Francisco, California, United States')."
+            },
+            "country": {
+                "type": "string",
+                "description": "ISO country code. Default 'US'."
+            },
+            "includeDomains": {
+                "type": "array",
+                "description": "Only return results from these hostnames (mutually exclusive with excludeDomains)."
+            },
+            "excludeDomains": {
+                "type": "array",
+                "description": "Exclude results from these hostnames (mutually exclusive with includeDomains)."
+            },
+            "timeout": {
+                "type": "integer",
+                "description": "Request timeout in milliseconds. Default 60000."
+            },
+            "scrapeOptions": {
+                "type": "object",
+                "description": "When provided, each result is also scraped and its content returned. Accepts the same content options as /scrape (e.g. `formats`, `onlyMainContent`). Increases cost because every result is fetched."
+            }
+        },
+        "required": [
+            "query"
+        ]
+    },
+    execute: async (input) => {
+      const key = await ensureApiKey()
+      const params = parseToolInput(input)
+      return callSuperagnt(key, 'POST', '/search', undefined, params)
+    },
+  })
+
+  api.registerTool({
+    name: 'superagnt_web_map',
+    description: "Map a website's URLs",
+    inputSchema: {
+        "type": "object",
+        "properties": {
+            "url": {
+                "type": "string",
+                "description": "The base URL of the site to map."
+            },
+            "search": {
+                "type": "string",
+                "description": "Order returned URLs by relevance to this query."
+            },
+            "sitemap": {
+                "type": "string",
+                "description": "How to use the site's sitemap. Default 'include'."
+            },
+            "includeSubdomains": {
+                "type": "boolean",
+                "description": "Include URLs on subdomains."
+            },
+            "ignoreQueryParameters": {
+                "type": "boolean",
+                "description": "Treat URLs that differ only by query string as the same URL."
+            },
+            "limit": {
+                "type": "integer",
+                "description": "Maximum number of URLs to return. Default 5000, max 100000."
+            },
+            "timeout": {
+                "type": "integer",
+                "description": "Request timeout in milliseconds."
+            },
+            "location": {
+                "type": "object",
+                "description": "Geographic proxy location and language settings."
+            }
+        },
+        "required": [
+            "url"
+        ]
+    },
+    execute: async (input) => {
+      const key = await ensureApiKey()
+      const params = parseToolInput(input)
+      return callSuperagnt(key, 'POST', '/map', undefined, params)
+    },
+  })
+
+  promptState.toolCount = 3
+  promptState.ready = true
+}

package/openclaw.plugin.json

@@ -0,0 +1,16 @@
+{
+  "id": "web-scraping",
+  "name": "Web Scraping",
+  "description": "Web scraping, search, and site mapping through the superagnt unified API.",
+  "version": "2.0.2",
+  "configSchema": {
+    "type": "object",
+    "properties": {
+      "apiKey": {
+        "type": "string",
+        "description": "Your superagnt API key. Get one at https://app.superagnt.com/dashboard or store in ~/.superagnt/config.json via CLI login."
+      }
+    },
+    "additionalProperties": false
+  }
+}

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@superagnt/web-scraping",
+  "version": "2.0.2",
+  "type": "module",
+  "description": "Web scraping, search, and site mapping through the superagnt unified API.",
+  "main": "index.js",
+  "license": "MIT",
+  "homepage": "https://superagnt.com/r/npm-web-scraping-docs",
+  "keywords": [
+    "openclaw",
+    "openclaw-plugin",
+    "superagnt",
+    "mcp",
+    "web",
+    "social-data",
+    "api",
+    "ai-agent",
+    "web scraping api",
+    "web search api for ai agents",
+    "scrape to markdown api",
+    "structured web extraction api",
+    "site map url discovery api",
+    "agent-ready web data api",
+    "mcp web data source"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "index.js",
+    "openclaw.plugin.json",
+    "README.md"
+  ],
+  "openclaw": {
+    "extensions": [
+      "./index.js"
+    ],
+    "compat": {
+      "pluginApi": ">=2026.3.24-beta.2",
+      "minGatewayVersion": "2026.3.24-beta.2"
+    },
+    "build": {
+      "openclawVersion": "2026.3.24-beta.2",
+      "pluginSdkVersion": "2026.3.24-beta.2"
+    }
+  }
+}