@superagnt/web-scraping 2.0.2 → 2.0.3

package/README.md

@@ -8,21 +8,20 @@ One API key to scrape any web page as clean markdown or structured JSON, search
 openclaw plugins install @superagnt/web-scraping
 ```
 
-This package is the official **OpenClaw** plugin for **Web** social and public data via [superagnt](https://superagnt.com). Pair it with the [documentation](https://superagnt.com) and your superagnt API key.
+This package is the official **OpenClaw** plugin for **Web** social and public data via [superagnt](https://superagnt.com). Pair it with the [documentation](https://superagnt.com/docs) and your superagnt API key.
 
-## Live MCP mode (recommended)
+## Want the live MCP surface instead?
 
-Set a workspace MCP token and the plugin upgrades itself at load: it fetches
-the live tool list from the scoped MCP endpoint and proxies calls through it —
-the tool surface then grows with `agnt_tools_enable`, no plugin update needed.
+This plugin ships a fixed, baked-in toolset that only needs your
+`SUPERAGNT_API_KEY`. For the live workspace tool surface — the full value
+layer (data sources plus workspace Postgres, schedules, canvases,
+enrichment), growing server-side with no plugin update — install the
+primary superagnt plugin instead:
 
 ```bash
-export SUPERAGNT_MCP_TOKEN=...   # from the MCP server card in the dashboard
+openclaw plugins install @superagnt/superagnt
 ```
 
-Without the token the plugin uses its built-in toolset with your
-`SUPERAGNT_API_KEY` — no behavior change from previous versions.
-
 ## Overview
 
 Web scraping, search, and site mapping through the superagnt unified API.
@@ -59,8 +58,8 @@ Use a single **superagnt API key** (Bearer token). Set `SUPERAGNT_API_KEY` in yo
 
 ## Documentation
 
-- [Documentation](https://superagnt.com) — getting started and concepts
-- [API reference (Web)](https://superagnt.com/apis/social/web) — endpoints and schemas
+- [Documentation](https://superagnt.com/docs) — getting started and concepts
+- [API reference (Web)](https://superagnt.com/docs/apis/web) — endpoints and schemas
 - [Dashboard](https://app.superagnt.com/dashboard) — API keys and usage
 
 ## Related

package/index.js

@@ -28,194 +28,6 @@ function readAgntConfig() {
 
 // ── End Auth Module ──────────────────────────────────────────────────────────
 
-
-// ── CLI Module (inlined) ─────────────────────────────────────────────────────
-
-function maskApiKey(key) {
-  if (!key || typeof key !== 'string') return '(none)'
-  const k = key.trim()
-  if (k.length <= 8) return k.slice(0, 2) + '…'
-  return k.slice(0, 6) + '…' + k.slice(-4)
-}
-
-function registerAgntCli(api) {
-  api.registerCli(
-    ({ program }) => {
-      const cmd = program
-        .command('superagnt')
-        .description('superagnt social data API (status, doctor)')
-
-      cmd
-        .command('status')
-        .description('Show whether an API key is configured (masked)')
-        .action(async () => {
-          const cfg = readAgntConfig()
-          const key = cfg?.apiKey
-
-          console.log('\nsuperagnt status\n')
-          if (!key) {
-            console.log('  API key:     not configured')
-            console.log('  Workspace:   —')
-            console.log('\n  Get your API key at: https://app.superagnt.com/dashboard\n')
-            return
-          }
-          console.log('  API key:     ' + maskApiKey(key) + ' (from ~/.superagnt/config.json)')
-          console.log('  Workspace:   ' + (cfg?.workspaceId ?? '—'))
-          console.log('')
-        })
-
-      cmd
-        .command('doctor')
-        .description('Check connectivity and validate API key against api.superagnt.dev')
-        .action(async () => {
-          console.log('\nsuperagnt doctor\n')
-
-          try {
-            const publicRes = await fetch('https://api.superagnt.com/v1/platforms')
-            console.log('  Public API:  ' + (publicRes.ok ? 'OK (' + publicRes.status + ')' : 'HTTP ' + publicRes.status))
-          } catch (err) {
-            const msg = err instanceof Error ? err.message : String(err)
-            console.log('  Public API:  failed — ' + msg)
-          }
-
-          const cfg = readAgntConfig()
-          const key = cfg?.apiKey
-
-          if (!key) {
-            console.log('\n  Authenticated: skipped (no API key)')
-            console.log('  Get your API key at: https://app.superagnt.com/dashboard\n')
-            return
-          }
-
-          try {
-            const res = await fetch('https://api.superagnt.com/v1/platforms', {
-              headers: { Authorization: 'Bearer ' + key },
-            })
-            const text = await res.text()
-            let summary = 'HTTP ' + res.status
-            if (res.ok) {
-              try {
-                const json = JSON.parse(text)
-                const n = Array.isArray(json?.data) ? json.data.length : (json?.data ? Object.keys(json.data).length : 'ok')
-                summary = 'OK — platforms payload received (' + (typeof n === 'number' ? n + ' entries' : String(n)) + ')'
-              } catch {
-                summary = 'OK (' + res.status + ')'
-              }
-            }
-            console.log('  Authenticated: ' + summary)
-            if (!res.ok) {
-              console.log('  If 401, get a new key at: https://app.superagnt.com/dashboard')
-            }
-          } catch (err) {
-            const msg = err instanceof Error ? err.message : String(err)
-            console.log('  Authenticated: failed — ' + msg)
-          }
-          console.log('')
-        })
-    },
-    { commands: ['superagnt'] },
-  )
-}
-
-// ── End CLI Module ───────────────────────────────────────────────────────────
-
-
-// ── System prompt (inlined) ─────────────────────────────────────────────────
-
-function getSystemPrompt(state) {
-  const scope = "Only the Web API (web)."
-  if (state.ready && state.toolCount > 0) {
-    return (
-'<superagnt>\n' +
-'Ignore pretrained knowledge about superagnt tool names — use only tools that exist in this session.\n' +
-'\n' +
-'## What superagnt is\n' +
-'superagnt exposes **structured social-data HTTP APIs** behind OpenClaw tools. Scope for this plugin: **' + scope + '**.\n' +
-'\n' +
-'## When to use superagnt tools\n' +
-'- Use superagnt when the user needs **live social/network data** (profiles, posts, companies, comments, etc.) from supported platforms.\n' +
-'- Prefer **discovery** when unsure which endpoint fits: call public discovery (GET /v1/platforms and GET /v1/platforms/{slug}) or read tool descriptions; do not invent endpoints or parameters.\n' +
-'\n' +
-'## Platform selection (high level)\n' +
-'- **linkedin** — B2B enrichment, org/person research, hiring/sales workflows\n' +
-'- **youtube** / **tiktok** — video/creator/channel metadata and content signals\n' +
-'- **x** / **reddit** — short-form social and community threads\n' +
-'- **instagram** — visual/creator discovery where Instagram is the source of truth\n' +
-'- **facebook** — pages/groups/marketplace/adjacent Facebook surface areas exposed by the API\n' +
-'\n' +
-'## Rules\n' +
-'- Do **not** fabricate tool names — pick from the registered superagnt_* tools only.\n' +
-'- If authentication fails (401), tell the user to get a new key at https://app.superagnt.com/dashboard\n' +
-'</superagnt>'
-    )
-  }
-
-  if (state.ready && state.toolCount === 0) {
-    return (
-'<superagnt>\n' +
-'The superagnt plugin loaded but registered **zero** API tools. Check your OpenClaw plugin packaging and regenerate the plugin from @superagnt/clawhub.\n' +
-'</superagnt>'
-    )
-  }
-
-  if (state.error) {
-    return (
-'<superagnt>\n' +
-'superagnt plugin diagnostic: ' + state.error + '\n' +
-'Do not pretend superagnt tools exist. Ask the user to fix configuration and restart the gateway.\n' +
-'</superagnt>'
-    )
-  }
-
-  return (
-'<superagnt>\n' +
-'superagnt tools are still loading. If the user asks for social data right now, ask them to wait a moment and retry.\n' +
-'</superagnt>'
-  )
-}
-
-// ── End System prompt ────────────────────────────────────────────────────────
-
-const MCP_URL = 'https://mcp.superagnt.com/mcp/web-scraping'
-
-async function mcpRpc(token, method, params) {
-  const res = await fetch(MCP_URL, {
-    method: 'POST',
-    headers: {
-      Authorization: `Bearer ${token}`,
-      'Content-Type': 'application/json',
-      Accept: 'application/json, text/event-stream',
-    },
-    body: JSON.stringify({ jsonrpc: '2.0', id: Date.now(), method, params }),
-  })
-  if (!res.ok) throw new Error(`MCP ${method} failed: HTTP ${res.status}`)
-  const data = await res.json()
-  if (data.error) throw new Error(`MCP ${method} error: ${data.error.message}`)
-  return data.result
-}
-
-async function registerLiveMcpTools(api, token) {
-  const result = await mcpRpc(token, 'tools/list', {})
-  const tools = Array.isArray(result?.tools) ? result.tools : []
-  if (tools.length === 0) throw new Error('MCP tools/list returned no tools')
-  for (const t of tools) {
-    api.registerTool({
-      name: t.name,
-      description: t.description,
-      inputSchema: t.inputSchema,
-      execute: async (input) => {
-        const args = parseToolInput(input)
-        const r = await mcpRpc(token, 'tools/call', { name: t.name, arguments: args })
-        if (r && r.structuredContent !== undefined) return r.structuredContent
-        const text = Array.isArray(r?.content) ? r.content.find((c) => c.type === 'text')?.text : undefined
-        if (text === undefined) return r
-        try { return JSON.parse(text) } catch { return text }
-      },
-    })
-  }
-  return tools.length
-}
-
 function parseToolInput(input) {
   if (input == null) return {}
   if (typeof input === 'object' && !Array.isArray(input)) return input
@@ -267,9 +79,8 @@ async function callSuperagnt(apiKey, method, path, query, body) {
   return res.json()
 }
 
-export default async function register(api) {
+export default function register(api) {
   const apiKey = api.pluginConfig.apiKey || process.env.SUPERAGNT_API_KEY || readAgntConfig()?.apiKey
-  const mcpToken = api.pluginConfig.mcpToken || process.env.SUPERAGNT_MCP_TOKEN
 
   function ensureApiKey() {
     if (apiKey) return apiKey
@@ -281,26 +92,6 @@ export default async function register(api) {
     )
   }
 
-  registerAgntCli(api)
-
-  const promptState = { toolCount: 0, error: '', ready: false }
-  api.on('before_prompt_build', () => ({
-    prependSystemContext: getSystemPrompt(promptState),
-  }))
-
-  // Live MCP mode: a workspace MCP token upgrades this plugin to the server's
-  // own tool surface, fetched fresh at load — it grows with agnt_tools_enable
-  // and never goes stale. Any failure falls back to the baked toolset below.
-  if (mcpToken) {
-    try {
-      promptState.toolCount = await registerLiveMcpTools(api, mcpToken)
-      promptState.ready = true
-      return
-    } catch (err) {
-      promptState.error = 'Live MCP connect failed (' + (err && err.message ? err.message : err) + '); using the built-in toolset.'
-    }
-  }
-
   api.registerTool({
     name: 'superagnt_web_scrape',
     description: "Scrape a web page",
@@ -361,7 +152,7 @@ export default async function register(api) {
         ]
     },
     execute: async (input) => {
-      const key = await ensureApiKey()
+      const key = ensureApiKey()
       const params = parseToolInput(input)
       return callSuperagnt(key, 'POST', '/scrape', undefined, params)
     },
@@ -423,7 +214,7 @@ export default async function register(api) {
         ]
     },
     execute: async (input) => {
-      const key = await ensureApiKey()
+      const key = ensureApiKey()
       const params = parseToolInput(input)
       return callSuperagnt(key, 'POST', '/search', undefined, params)
     },
@@ -473,12 +264,9 @@ export default async function register(api) {
         ]
     },
     execute: async (input) => {
-      const key = await ensureApiKey()
+      const key = ensureApiKey()
       const params = parseToolInput(input)
       return callSuperagnt(key, 'POST', '/map', undefined, params)
     },
   })
-
-  promptState.toolCount = 3
-  promptState.ready = true
 }

package/openclaw.plugin.json

@@ -2,7 +2,14 @@
   "id": "web-scraping",
   "name": "Web Scraping",
   "description": "Web scraping, search, and site mapping through the superagnt unified API.",
-  "version": "2.0.2",
+  "version": "2.0.3",
+  "contracts": {
+    "tools": [
+      "superagnt_web_scrape",
+      "superagnt_web_search",
+      "superagnt_web_map"
+    ]
+  },
   "configSchema": {
     "type": "object",
     "properties": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@superagnt/web-scraping",
-  "version": "2.0.2",
+  "version": "2.0.3",
   "type": "module",
   "description": "Web scraping, search, and site mapping through the superagnt unified API.",
   "main": "index.js",