mcp-scraper 0.3.6 → 0.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -26
- package/dist/bin/api-server.cjs +158 -321
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs +39 -1750
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -4
- package/dist/bin/browser-agent-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-cli.cjs +47 -328
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +7 -91
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +78 -1796
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +4 -6
- package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +4 -4
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +18 -25
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/chunk-2NEVRPK3.js +7 -0
- package/dist/chunk-2NEVRPK3.js.map +1 -0
- package/dist/{chunk-ROZEDHAA.js → chunk-2O7RAK6E.js} +19 -26
- package/dist/chunk-2O7RAK6E.js.map +1 -0
- package/dist/{chunk-RVYHP7QN.js → chunk-ELI6NDEG.js} +4 -4
- package/dist/{chunk-RVYHP7QN.js.map → chunk-ELI6NDEG.js.map} +1 -1
- package/dist/{chunk-TK4D2BZ5.js → chunk-IWNLHCKA.js} +13 -694
- package/dist/chunk-IWNLHCKA.js.map +1 -0
- package/dist/{server-YSKBD2VX.js → server-HRVD52CX.js} +836 -114
- package/dist/server-HRVD52CX.js.map +1 -0
- package/docs/mcp-tool-craft-lint.generated.md +2 -5
- package/docs/mcp-tool-manifest.generated.json +9 -101
- package/package.json +1 -1
- package/dist/chunk-DUEW4EOO.js +0 -214
- package/dist/chunk-DUEW4EOO.js.map +0 -1
- package/dist/chunk-IHXHQL53.js +0 -7
- package/dist/chunk-IHXHQL53.js.map +0 -1
- package/dist/chunk-NXRWFOEZ.js +0 -816
- package/dist/chunk-NXRWFOEZ.js.map +0 -1
- package/dist/chunk-ROZEDHAA.js.map +0 -1
- package/dist/chunk-TK4D2BZ5.js.map +0 -1
- package/dist/server-YSKBD2VX.js.map +0 -1
|
@@ -2,21 +2,19 @@
|
|
|
2
2
|
import {
|
|
3
3
|
HttpMcpToolExecutor,
|
|
4
4
|
registerPaaExtractorMcpTools
|
|
5
|
-
} from "../chunk-
|
|
5
|
+
} from "../chunk-2O7RAK6E.js";
|
|
6
6
|
import "../chunk-M2S27J6Z.js";
|
|
7
7
|
import {
|
|
8
8
|
registerBrowserAgentMcpTools
|
|
9
|
-
} from "../chunk-
|
|
10
|
-
import "../chunk-NXRWFOEZ.js";
|
|
9
|
+
} from "../chunk-IWNLHCKA.js";
|
|
11
10
|
import "../chunk-WN7PBKMV.js";
|
|
12
11
|
import {
|
|
13
12
|
renderInstallTerminal
|
|
14
|
-
} from "../chunk-
|
|
13
|
+
} from "../chunk-ELI6NDEG.js";
|
|
15
14
|
import "../chunk-LFATOGDF.js";
|
|
16
|
-
import "../chunk-DUEW4EOO.js";
|
|
17
15
|
import {
|
|
18
16
|
PACKAGE_VERSION
|
|
19
|
-
} from "../chunk-
|
|
17
|
+
} from "../chunk-2NEVRPK3.js";
|
|
20
18
|
|
|
21
19
|
// bin/mcp-scraper-combined-stdio-server.ts
|
|
22
20
|
import { readFileSync } from "fs";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../bin/mcp-scraper-combined-stdio-server.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { readFileSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'\nimport { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'\nimport { HttpMcpToolExecutor } from '../src/mcp/http-mcp-tool-executor.js'\nimport { registerBrowserAgentMcpTools } from '../src/mcp/browser-agent-mcp-server.js'\nimport { registerPaaExtractorMcpTools } from '../src/mcp/paa-mcp-server.js'\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\n\nconst forceStdio =\n process.argv.includes('--stdio') ||\n process.env.MCP_SCRAPER_FORCE_STDIO === '1'\nconst interactiveTerminal = Boolean(process.stdin.isTTY && process.stdout.isTTY)\nconst wantsHelp = process.argv.includes('--help') || process.argv.includes('-h')\n\nif (!forceStdio && (interactiveTerminal || wantsHelp)) {\n const noColor =\n process.argv.includes('--no-color') ||\n process.env.NO_COLOR !== undefined ||\n process.env.FORCE_COLOR === '0' ||\n !process.stdout.isTTY\n\n process.stdout.write(renderInstallTerminal({\n version: PACKAGE_VERSION,\n color: !noColor,\n apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n }))\n process.exit(0)\n}\n\nfunction readApiKeyFile(): string | undefined {\n const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim()\n const paths = [explicitPath, join(homedir(), '.mcp-scraper-key')].filter(Boolean) as string[]\n for (const path of paths) {\n try {\n const value = readFileSync(path, 'utf8').trim()\n if (value) return value\n } catch {\n void 0\n }\n }\n return undefined\n}\n\nconst apiKey = (\n process.env.MCP_SCRAPER_API_KEY ??\n process.env.MCP_SCRAPER_KEY ??\n process.env.MCP_API_KEY ??\n readApiKeyFile()\n)?.trim()\nif (!apiKey) {\n process.stderr.write('MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\\n')\n process.exit(1)\n}\n\nconst baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || 'https://mcpscraper.dev'\nconst consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl\nconst server = new McpServer({ name: 'mcp-scraper', version: PACKAGE_VERSION })\n\nregisterPaaExtractorMcpTools(server, new HttpMcpToolExecutor(baseUrl, apiKey))\nregisterBrowserAgentMcpTools(server, { baseUrl, apiKey, consoleBaseUrl })\n\nconst transport = new StdioServerTransport()\n\nasync function main() {\n await server.connect(transport)\n}\n\nmain().catch(err => {\n process.stderr.write(`${err instanceof Error ? err.message : String(err)}\\n`)\n process.exit(1)\n})\n"],"mappings":"
|
|
1
|
+
{"version":3,"sources":["../../bin/mcp-scraper-combined-stdio-server.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { readFileSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'\nimport { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'\nimport { HttpMcpToolExecutor } from '../src/mcp/http-mcp-tool-executor.js'\nimport { registerBrowserAgentMcpTools } from '../src/mcp/browser-agent-mcp-server.js'\nimport { registerPaaExtractorMcpTools } from '../src/mcp/paa-mcp-server.js'\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\n\nconst forceStdio =\n process.argv.includes('--stdio') ||\n process.env.MCP_SCRAPER_FORCE_STDIO === '1'\nconst interactiveTerminal = Boolean(process.stdin.isTTY && process.stdout.isTTY)\nconst wantsHelp = process.argv.includes('--help') || process.argv.includes('-h')\n\nif (!forceStdio && (interactiveTerminal || wantsHelp)) {\n const noColor =\n process.argv.includes('--no-color') ||\n process.env.NO_COLOR !== undefined ||\n process.env.FORCE_COLOR === '0' ||\n !process.stdout.isTTY\n\n process.stdout.write(renderInstallTerminal({\n version: PACKAGE_VERSION,\n color: !noColor,\n apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n }))\n process.exit(0)\n}\n\nfunction readApiKeyFile(): string | undefined {\n const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim()\n const paths = [explicitPath, join(homedir(), '.mcp-scraper-key')].filter(Boolean) as string[]\n for (const path of paths) {\n try {\n const value = readFileSync(path, 'utf8').trim()\n if (value) return value\n } catch {\n void 0\n }\n }\n return undefined\n}\n\nconst apiKey = (\n process.env.MCP_SCRAPER_API_KEY ??\n process.env.MCP_SCRAPER_KEY ??\n process.env.MCP_API_KEY ??\n readApiKeyFile()\n)?.trim()\nif (!apiKey) {\n process.stderr.write('MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\\n')\n process.exit(1)\n}\n\nconst baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || 'https://mcpscraper.dev'\nconst consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl\nconst server = new McpServer({ name: 'mcp-scraper', version: PACKAGE_VERSION })\n\nregisterPaaExtractorMcpTools(server, new HttpMcpToolExecutor(baseUrl, apiKey))\nregisterBrowserAgentMcpTools(server, { baseUrl, apiKey, consoleBaseUrl })\n\nconst transport = new StdioServerTransport()\n\nasync function main() {\n await server.connect(transport)\n}\n\nmain().catch(err => {\n process.stderr.write(`${err instanceof Error ? err.message : String(err)}\\n`)\n process.exit(1)\n})\n"],"mappings":";;;;;;;;;;;;;;;;;;;AACA,SAAS,oBAAoB;AAC7B,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,iBAAiB;AAC1B,SAAS,4BAA4B;AAOrC,IAAM,aACJ,QAAQ,KAAK,SAAS,SAAS,KAC/B,QAAQ,IAAI,4BAA4B;AAC1C,IAAM,sBAAsB,QAAQ,QAAQ,MAAM,SAAS,QAAQ,OAAO,KAAK;AAC/E,IAAM,YAAY,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAE/E,IAAI,CAAC,eAAe,uBAAuB,YAAY;AACrD,QAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,UAAQ,OAAO,MAAM,sBAAsB;AAAA,IACzC,SAAS;AAAA,IACT,OAAO,CAAC;AAAA,IACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AAAA,EACnE,CAAC,CAAC;AACF,UAAQ,KAAK,CAAC;AAChB;AAEA,SAAS,iBAAqC;AAC5C,QAAM,eAAe,QAAQ,IAAI,sBAAsB,KAAK;AAC5D,QAAM,QAAQ,CAAC,cAAc,KAAK,QAAQ,GAAG,kBAAkB,CAAC,EAAE,OAAO,OAAO;AAChF,aAAW,QAAQ,OAAO;AACxB,QAAI;AACF,YAAM,QAAQ,aAAa,MAAM,MAAM,EAAE,KAAK;AAC9C,UAAI,MAAO,QAAO;AAAA,IACpB,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAEA,IAAM,UACJ,QAAQ,IAAI,uBACZ,QAAQ,IAAI,mBACZ,QAAQ,IAAI,eACZ,eAAe,IACd,KAAK;AACR,IAAI,CAAC,QAAQ;AACX,UAAQ,OAAO,MAAM,iEAAiE;AACtF,UAAQ,KAAK,CAAC;AAChB;AAEA,IAAM,UAAU,QAAQ,IAAI,sBAAsB,KAAK,KAAK,QAAQ,IAAI,cAAc,KAAK,KAAK;AAChG,IAAM,iBAAiB,QAAQ,IAAI,2BAA2B,KAAK,KAAK;AACxE,IAAM,SAAS,IAAI,UAAU,EAAE,MAAM,eAAe,SAAS,gBAAgB,CAAC;AAE9E,6BAA6B,QAAQ,IAAI,oBAAoB,SAAS,MAAM,CAAC;AAC7E,6BAA6B,QAAQ,EAAE,SAAS,QAAQ,eAAe,CAAC;AAExE,IAAM,YAAY,IAAI,qBAAqB;AAE3C,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,SAAS;AAChC;AAEA,KAAK,EAAE,MAAM,SAAO;AAClB,UAAQ,OAAO,MAAM,GAAG,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,CAAI;AAC5E,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}
|
|
@@ -53,16 +53,16 @@ function renderInstallTerminal(options) {
|
|
|
53
53
|
"1/1 install surfaces ready",
|
|
54
54
|
colorize("Newest: saved hosted browser profiles for AI visibility. Open Google, share the watch_url, user signs in, close to save. Browser sessions are direct/no-proxy by default.", "lime", color),
|
|
55
55
|
"",
|
|
56
|
-
`${colorize("Tools", "cyan", color)} ${colorize("(
|
|
56
|
+
`${colorize("Tools", "cyan", color)} ${colorize("(43 MCP tools)", "muted", color)}`,
|
|
57
57
|
toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
|
|
58
58
|
toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "directory_workflow"], color),
|
|
59
59
|
toolRow("build", ["rank_tracker_blueprint", "cron plan", "database prompt"], color),
|
|
60
60
|
toolRow("media", ["youtube_harvest", "youtube_transcribe", "facebook_ad_search", "facebook_page_intel", "facebook_ad_transcribe", "facebook_video_transcribe", "instagram_profile_content", "instagram_media_download"], color),
|
|
61
61
|
toolRow("browser", ["browser_open", "browser_profile_onboard", "browser_close", "browser_screenshot", "browser_read", "browser_locate", "browser_replay_mark", "browser_replay_annotate"], color),
|
|
62
|
-
toolRow("account", ["credits_info", "
|
|
62
|
+
toolRow("account", ["credits_info", "reports", "MCP resources"], color),
|
|
63
63
|
`${colorize("Workflows", "cyan", color)} ${colorize("(MCP + CLI + API)", "muted", color)}`,
|
|
64
64
|
toolRow("route", ["workflow_list", "workflow_suggest", "workflow_run", "workflow_step", "workflow_status", "workflow_artifact_read"], color),
|
|
65
|
-
toolRow("seo", ["directory", "agent-packet", "
|
|
65
|
+
toolRow("seo", ["directory", "agent-packet", "competitive audit", "map/serp comparison", "PAA/AIO briefs", "scheduled runs"], color),
|
|
66
66
|
"",
|
|
67
67
|
colorize("Usage tips:", "amber", color),
|
|
68
68
|
"Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.",
|
|
@@ -101,7 +101,7 @@ function renderInstallTerminal(options) {
|
|
|
101
101
|
}
|
|
102
102
|
|
|
103
103
|
// src/version.ts
|
|
104
|
-
var PACKAGE_VERSION = "0.3.
|
|
104
|
+
var PACKAGE_VERSION = "0.3.7";
|
|
105
105
|
|
|
106
106
|
// bin/mcp-scraper-install.ts
|
|
107
107
|
var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/install-terminal.ts","../../src/version.ts","../../bin/mcp-scraper-install.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n color?: boolean\n version: string\n apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n reset: '\\x1b[0m',\n cyan: '\\x1b[36m',\n lime: '\\x1b[32m',\n amber: '\\x1b[33m',\n red: '\\x1b[31m',\n muted: '\\x1b[90m',\n bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n if (!enabled) return value\n return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n const padded = label.padEnd(9, ' ')\n return ` ${colorize(padded, 'muted', enabled)} ${tools.join(colorize(' . ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n const color = options.color ?? true\n const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n const ascii = String.raw`\n __ __ ____ ____ \n| \\/ |/ ___| _ \\\n| |\\/| | | | |_) |\n| | | | |___| __/\n|_| |_|\\____|_|\n\n ____ ____ ____ _ ____ _____ ____\n/ ___| / ___| _ \\ / \\ | _ \\| ____| _ \\\n\\___ \\| | | |_) | / _ \\ | |_) | _| | |_) |\n ___) | |___| _ < / ___ \\| __/| |___| _ <\n|____/ \\____|_| \\_\\/_/ \\_\\_| |_____|_| \\_\\\n`\n\n const claudeCommand = [\n `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n ' mcp-scraper-cli agent install claude --apply',\n ].join('\\n')\n\n const codexConfig = [\n '[mcp_servers.mcp-scraper]',\n 'command = \"npx\"',\n 'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper-combined\"]',\n `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n ].join('\\n')\n\n return [\n colorize(`mcp-scraper v${options.version}`, 'bold', color),\n colorize('> mcp-scraper-install', 'muted', color),\n colorize(ascii, 'amber', color),\n `${colorize('MCP Scraper Agent', 'cyan', color)} . v${options.version} . mcpscraper.dev`,\n '1/1 install surfaces ready',\n colorize('Newest: saved hosted browser profiles for AI visibility. Open Google, share the watch_url, user signs in, close to save. Browser sessions are direct/no-proxy by default.', 'lime', color),\n '',\n `${colorize('Tools', 'cyan', color)} ${colorize('(
|
|
1
|
+
{"version":3,"sources":["../../src/install-terminal.ts","../../src/version.ts","../../bin/mcp-scraper-install.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n color?: boolean\n version: string\n apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n reset: '\\x1b[0m',\n cyan: '\\x1b[36m',\n lime: '\\x1b[32m',\n amber: '\\x1b[33m',\n red: '\\x1b[31m',\n muted: '\\x1b[90m',\n bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n if (!enabled) return value\n return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n const padded = label.padEnd(9, ' ')\n return ` ${colorize(padded, 'muted', enabled)} ${tools.join(colorize(' . ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n const color = options.color ?? true\n const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n const ascii = String.raw`\n __ __ ____ ____ \n| \\/ |/ ___| _ \\\n| |\\/| | | | |_) |\n| | | | |___| __/\n|_| |_|\\____|_|\n\n ____ ____ ____ _ ____ _____ ____\n/ ___| / ___| _ \\ / \\ | _ \\| ____| _ \\\n\\___ \\| | | |_) | / _ \\ | |_) | _| | |_) |\n ___) | |___| _ < / ___ \\| __/| |___| _ <\n|____/ \\____|_| \\_\\/_/ \\_\\_| |_____|_| \\_\\\n`\n\n const claudeCommand = [\n `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n ' mcp-scraper-cli agent install claude --apply',\n ].join('\\n')\n\n const codexConfig = [\n '[mcp_servers.mcp-scraper]',\n 'command = \"npx\"',\n 'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper-combined\"]',\n `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n ].join('\\n')\n\n return [\n colorize(`mcp-scraper v${options.version}`, 'bold', color),\n colorize('> mcp-scraper-install', 'muted', color),\n colorize(ascii, 'amber', color),\n `${colorize('MCP Scraper Agent', 'cyan', color)} . v${options.version} . mcpscraper.dev`,\n '1/1 install surfaces ready',\n colorize('Newest: saved hosted browser profiles for AI visibility. Open Google, share the watch_url, user signs in, close to save. Browser sessions are direct/no-proxy by default.', 'lime', color),\n '',\n `${colorize('Tools', 'cyan', color)} ${colorize('(43 MCP tools)', 'muted', color)}`,\n toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'directory_workflow'], color),\n toolRow('build', ['rank_tracker_blueprint', 'cron plan', 'database prompt'], color),\n toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel', 'facebook_ad_transcribe', 'facebook_video_transcribe', 'instagram_profile_content', 'instagram_media_download'], color),\n toolRow('browser', ['browser_open', 'browser_profile_onboard', 'browser_close', 'browser_screenshot', 'browser_read', 'browser_locate', 'browser_replay_mark', 'browser_replay_annotate'], color),\n toolRow('account', ['credits_info', 'reports', 'MCP resources'], color),\n `${colorize('Workflows', 'cyan', color)} ${colorize('(MCP + CLI + API)', 'muted', color)}`,\n toolRow('route', ['workflow_list', 'workflow_suggest', 'workflow_run', 'workflow_step', 'workflow_status', 'workflow_artifact_read'], color),\n toolRow('seo', ['directory', 'agent-packet', 'competitive audit', 'map/serp comparison', 'PAA/AIO briefs', 'scheduled runs'], color),\n '',\n colorize('Usage tips:', 'amber', color),\n 'Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.',\n 'Run mcp-scraper-combined in a human terminal to print this card; MCP clients get the same command as a silent stdio server.',\n 'Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install',\n 'Customer auth setup: run browser_profile_onboard at https://google.com/, send the watch_url, let the user click Sign in, then browser_close to save.',\n 'Reconnect setup: if a saved profile shows login controls later, run browser_profile_onboard again with the same profile name.',\n 'Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.',\n 'For MCP clients, use mcp-scraper-combined so one install can mix SERP, Maps, browser, reports, and saved MCP resources.',\n 'If you hit the concurrency limit, add an extra slot for $5/month with mcp-scraper-cli billing concurrency checkout.',\n '',\n `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one command:`,\n '',\n colorize('Setup doctor', 'amber', color),\n 'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n '',\n colorize('Hosted profile setup', 'amber', color),\n 'In your MCP client, call browser_profile_onboard with profile=\"seo-example-com\" and url=\"https://google.com/\".',\n 'Give the returned watch_url to the user. After they sign in, call browser_close to persist the saved hosted profile.',\n '',\n colorize('Claude Code one-command setup', 'amber', color),\n claudeCommand,\n 'Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list',\n '',\n colorize('Codex config', 'amber', color),\n codexConfig,\n '',\n colorize('Claude Desktop Extension', 'amber', color),\n 'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n '',\n colorize('Safety note:', 'muted', color),\n 'mcp-scraper-combined prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.',\n 'Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.',\n '',\n ].join('\\n')\n}\n","export const PACKAGE_VERSION = '0.3.7'\n","#!/usr/bin/env node\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\n\nconst noColor =\n process.argv.includes('--no-color') ||\n process.env.NO_COLOR !== undefined ||\n process.env.FORCE_COLOR === '0' ||\n !process.stdout.isTTY\n\nconst help = process.argv.includes('--help') || process.argv.includes('-h')\nif (help) {\n process.stdout.write([\n 'Usage: mcp-scraper-install [--no-color]',\n '',\n 'Prints the branded MCP Scraper terminal install card and copyable install commands.',\n 'mcp-scraper-combined prints the same card in a human terminal and runs as the MCP stdio server in clients.',\n '',\n ].join('\\n'))\n process.exit(0)\n}\n\nprocess.stdout.write(renderInstallTerminal({\n version: PACKAGE_VERSION,\n color: !noColor,\n apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n}))\n"],"mappings":";;;;AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB,uBAAuB,WAAW;AAAA,IAClC;AAAA,EACF,EAAE,KAAK,IAAI;AAEX,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,6KAA6K,QAAQ,KAAK;AAAA,IACnM;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,kBAAkB,SAAS,KAAK,CAAC;AAAA,IAClF,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,oBAAoB,GAAG,KAAK;AAAA,IAChG,QAAQ,SAAS,CAAC,0BAA0B,aAAa,iBAAiB,GAAG,KAAK;AAAA,IAClF,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,uBAAuB,0BAA0B,6BAA6B,6BAA6B,0BAA0B,GAAG,KAAK;AAAA,IAC9N,QAAQ,WAAW,CAAC,gBAAgB,2BAA2B,iBAAiB,sBAAsB,gBAAgB,kBAAkB,uBAAuB,yBAAyB,GAAG,KAAK;AAAA,IAChM,QAAQ,WAAW,CAAC,gBAAgB,WAAW,eAAe,GAAG,KAAK;AAAA,IACtE,GAAG,SAAS,aAAa,QAAQ,KAAK,CAAC,KAAK,SAAS,qBAAqB,SAAS,KAAK,CAAC;AAAA,IACzF,QAAQ,SAAS,CAAC,iBAAiB,oBAAoB,gBAAgB,iBAAiB,mBAAmB,wBAAwB,GAAG,KAAK;AAAA,IAC3I,QAAQ,OAAO,CAAC,aAAa,gBAAgB,qBAAqB,uBAAuB,kBAAkB,gBAAgB,GAAG,KAAK;AAAA,IACnI;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,wBAAwB,SAAS,KAAK;AAAA,IAC/C;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,iCAAiC,SAAS,KAAK;AAAA,IACxD;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;;;AC9GO,IAAM,kBAAkB;;;ACI/B,IAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,IAAM,OAAO,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAC1E,IAAI,MAAM;AACR,UAAQ,OAAO,MAAM;AAAA,IACnB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI,CAAC;AACZ,UAAQ,KAAK,CAAC;AAChB;AAEA,QAAQ,OAAO,MAAM,sBAAsB;AAAA,EACzC,SAAS;AAAA,EACT,OAAO,CAAC;AAAA,EACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AACnE,CAAC,CAAC;","names":[]}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
renderInstallTerminal
|
|
4
|
-
} from "../chunk-
|
|
4
|
+
} from "../chunk-ELI6NDEG.js";
|
|
5
5
|
import {
|
|
6
6
|
PACKAGE_VERSION
|
|
7
|
-
} from "../chunk-
|
|
7
|
+
} from "../chunk-2NEVRPK3.js";
|
|
8
8
|
|
|
9
9
|
// bin/mcp-scraper-install.ts
|
|
10
10
|
var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
|
|
@@ -255,7 +255,7 @@ var import_node_fs2 = require("fs");
|
|
|
255
255
|
var import_node_path2 = require("path");
|
|
256
256
|
|
|
257
257
|
// src/version.ts
|
|
258
|
-
var PACKAGE_VERSION = "0.3.
|
|
258
|
+
var PACKAGE_VERSION = "0.3.7";
|
|
259
259
|
|
|
260
260
|
// src/mcp/mcp-response-formatter.ts
|
|
261
261
|
var import_node_fs = require("fs");
|
|
@@ -1713,16 +1713,13 @@ ${chunkRows}` : "",
|
|
|
1713
1713
|
}
|
|
1714
1714
|
function structuredInstagramBrowser(raw) {
|
|
1715
1715
|
const browser = raw && typeof raw === "object" ? raw : {};
|
|
1716
|
-
const mode = browser.mode === "local" ? "local" : "hosted";
|
|
1717
|
-
const requestedMode = browser.requestedMode === "local" || browser.requestedMode === "hosted" || browser.requestedMode === "auto" ? browser.requestedMode : "auto";
|
|
1718
|
-
const profileSource = browser.profileSource === "managed_profile" || browser.profileSource === "direct_profile_dir" ? browser.profileSource : "hosted";
|
|
1719
1716
|
return {
|
|
1720
|
-
mode,
|
|
1721
|
-
requestedMode,
|
|
1717
|
+
mode: "hosted",
|
|
1718
|
+
requestedMode: "hosted",
|
|
1722
1719
|
profileName: typeof browser.profileName === "string" ? browser.profileName : null,
|
|
1723
|
-
profileSource,
|
|
1724
|
-
profileDirConfigured:
|
|
1725
|
-
executablePathConfigured:
|
|
1720
|
+
profileSource: "hosted",
|
|
1721
|
+
profileDirConfigured: false,
|
|
1722
|
+
executablePathConfigured: false
|
|
1726
1723
|
};
|
|
1727
1724
|
}
|
|
1728
1725
|
function structuredInstagramPagination(raw, input) {
|
|
@@ -1764,7 +1761,7 @@ function formatInstagramProfileContent(raw, input) {
|
|
|
1764
1761
|
const itemRows = items.slice(0, 100).map(
|
|
1765
1762
|
(item, i) => `| ${i + 1} | ${item.type} | \`${item.shortcode}\` | ${item.url} | ${cell(item.firstSeenStage ?? "")} |`
|
|
1766
1763
|
).join("\n");
|
|
1767
|
-
const browserLabel =
|
|
1764
|
+
const browserLabel = "hosted browser";
|
|
1768
1765
|
const full = [
|
|
1769
1766
|
`# Instagram Profile Content: ${d.handle ?? input.handle ?? input.url ?? "profile"}`,
|
|
1770
1767
|
`**Collected:** ${items.length} items \xB7 posts ${typeCounts?.post ?? 0} \xB7 reels ${typeCounts?.reel ?? 0} \xB7 tv ${typeCounts?.tv ?? 0}`,
|
|
@@ -1839,7 +1836,7 @@ function formatInstagramMediaDownload(raw, input) {
|
|
|
1839
1836
|
const transcriptText = transcript?.text ?? "";
|
|
1840
1837
|
const chunks = transcript?.chunks ?? [];
|
|
1841
1838
|
const browser = structuredInstagramBrowser(d.browser);
|
|
1842
|
-
const browserLabel =
|
|
1839
|
+
const browserLabel = "hosted browser";
|
|
1843
1840
|
const downloadRows = downloads.map((download, i) => {
|
|
1844
1841
|
const status = download.error ? `error: ${cell(download.error)}` : `${download.sizeBytes ?? 0} bytes`;
|
|
1845
1842
|
return `| ${i + 1} | ${download.kind} | ${download.savedPath ? `\`${download.savedPath}\`` : "\u2014"} | ${status} |`;
|
|
@@ -2082,14 +2079,12 @@ var FacebookVideoTranscribeInputSchema = {
|
|
|
2082
2079
|
quality: import_zod2.z.enum(["best", "hd", "sd"]).default("best").describe("Preferred progressive MP4 quality. Use best by default; hd prefers the highest HD progressive URL; sd forces the SD URL.")
|
|
2083
2080
|
};
|
|
2084
2081
|
var InstagramProfileContentInputSchema = {
|
|
2085
|
-
handle: import_zod2.z.string().min(1).optional().describe("Instagram handle, with or without @. Provide handle or url.
|
|
2082
|
+
handle: import_zod2.z.string().min(1).optional().describe("Instagram handle, with or without @. Provide handle or url."),
|
|
2086
2083
|
url: import_zod2.z.string().url().optional().describe("Instagram profile URL, e.g. https://www.instagram.com/nasaartemis/. Provide handle or url."),
|
|
2087
|
-
maxItems: import_zod2.z.number().int().min(1).max(500).default(50).describe("Maximum profile grid post/reel/tv URLs to collect. Default 50, maximum 500. Use higher values
|
|
2088
|
-
maxScrolls: import_zod2.z.number().int().min(0).max(100).default(10).describe("Maximum pagination scroll attempts. Default 10, maximum 100. Increase
|
|
2084
|
+
maxItems: import_zod2.z.number().int().min(1).max(500).default(50).describe("Maximum profile grid post/reel/tv URLs to collect. Default 50, maximum 500. Use higher values only when the user asks for a fuller archive."),
|
|
2085
|
+
maxScrolls: import_zod2.z.number().int().min(0).max(100).default(10).describe("Maximum pagination scroll attempts. Default 10, maximum 100. Increase for long profiles when Instagram continues loading more grid links."),
|
|
2089
2086
|
scrollDelayMs: import_zod2.z.number().int().min(250).max(5e3).default(1200).describe("Delay after each pagination scroll before collecting newly loaded links. Default 1200ms. Increase to 2000-3000ms when Instagram loads slowly."),
|
|
2090
|
-
stableScrollLimit: import_zod2.z.number().int().min(1).max(10).default(4).describe("Stop after this many consecutive scrolls with no new links or scroll progress. Default 4.")
|
|
2091
|
-
browserMode: import_zod2.z.enum(["auto", "hosted", "local"]).default("auto").describe("Browser backend. auto uses the hosted browser unless local mode is configured or browserProfile is passed. local uses an imported/authenticated Chrome profile from browser_profile_import/sync. hosted forces the remote browser path."),
|
|
2092
|
-
browserProfile: import_zod2.z.string().min(1).optional().describe("Managed local browser profile name created by browser_profile_import/sync. Passing this opts the Instagram tool into local authenticated Chromium for this call.")
|
|
2087
|
+
stableScrollLimit: import_zod2.z.number().int().min(1).max(10).default(4).describe("Stop after this many consecutive scrolls with no new links or scroll progress. Default 4.")
|
|
2093
2088
|
};
|
|
2094
2089
|
var InstagramMediaDownloadInputSchema = {
|
|
2095
2090
|
url: import_zod2.z.string().url().describe("Instagram post, reel, or tv URL, e.g. https://www.instagram.com/reel/SHORTCODE/. The tool renders the page, extracts text, image metadata, and Instagram CDN media tracks."),
|
|
@@ -2097,9 +2092,7 @@ var InstagramMediaDownloadInputSchema = {
|
|
|
2097
2092
|
downloadMedia: import_zod2.z.boolean().default(true).describe("Download extracted text/media files to the MCP Scraper output directory when the API server can write files. Always returns extracted media URLs even when false."),
|
|
2098
2093
|
downloadAllTracks: import_zod2.z.boolean().default(false).describe("Download every captured Instagram MP4 track instead of only the selected best video and audio tracks. Use false by default to avoid duplicate bitrates."),
|
|
2099
2094
|
includeTranscript: import_zod2.z.boolean().default(false).describe("Transcribe the selected audio track when available. This adds transcription cost and may take longer."),
|
|
2100
|
-
mux: import_zod2.z.boolean().default(true).describe("When video and audio tracks are downloaded separately, try to mux them into a single MP4 if ffmpeg is available. Returns separate tracks when muxing is unavailable.")
|
|
2101
|
-
browserMode: import_zod2.z.enum(["auto", "hosted", "local"]).default("auto").describe("Browser backend. auto uses the hosted browser unless local mode is configured or browserProfile is passed. local uses an imported/authenticated Chrome profile from browser_profile_import/sync. hosted forces the remote browser path."),
|
|
2102
|
-
browserProfile: import_zod2.z.string().min(1).optional().describe("Managed local browser profile name created by browser_profile_import/sync. Passing this opts the Instagram tool into local authenticated Chromium for this call.")
|
|
2095
|
+
mux: import_zod2.z.boolean().default(true).describe("When video and audio tracks are downloaded separately, try to mux them into a single MP4 if ffmpeg is available. Returns separate tracks when muxing is unavailable.")
|
|
2103
2096
|
};
|
|
2104
2097
|
var MapsPlaceIntelInputSchema = {
|
|
2105
2098
|
businessName: import_zod2.z.string().min(1).describe('Business name only. If user says "Elite Roofing Denver CO", use businessName="Elite Roofing" and location="Denver, CO".'),
|
|
@@ -2498,10 +2491,10 @@ var TranscriptChunkOutput = import_zod2.z.object({
|
|
|
2498
2491
|
text: import_zod2.z.string()
|
|
2499
2492
|
});
|
|
2500
2493
|
var InstagramBrowserOutput = import_zod2.z.object({
|
|
2501
|
-
mode: import_zod2.z.
|
|
2502
|
-
requestedMode: import_zod2.z.
|
|
2494
|
+
mode: import_zod2.z.literal("hosted"),
|
|
2495
|
+
requestedMode: import_zod2.z.literal("hosted"),
|
|
2503
2496
|
profileName: NullableString,
|
|
2504
|
-
profileSource: import_zod2.z.
|
|
2497
|
+
profileSource: import_zod2.z.literal("hosted"),
|
|
2505
2498
|
profileDirConfigured: import_zod2.z.boolean(),
|
|
2506
2499
|
executablePathConfigured: import_zod2.z.boolean()
|
|
2507
2500
|
});
|
|
@@ -3243,14 +3236,14 @@ function registerPaaExtractorMcpTools(server2, executor2, options = {}) {
|
|
|
3243
3236
|
}, async (input) => formatFacebookVideoTranscribe(await executor2.facebookVideoTranscribe(input), input));
|
|
3244
3237
|
server2.registerTool("instagram_profile_content", {
|
|
3245
3238
|
title: "Instagram Profile Content Discovery",
|
|
3246
|
-
description: withReportNote("Discover Instagram profile grid content links for a handle or profile URL. Use this when the user wants a person or brand account content inventory before selecting posts/reels to download. Returns profile stats, collected post/reel/tv URLs, shortcodes, type counts, browser
|
|
3239
|
+
description: withReportNote("Discover Instagram profile grid content links for a handle or profile URL. Use this when the user wants a person or brand account content inventory before selecting posts/reels to download. Returns profile stats, collected post/reel/tv URLs, shortcodes, type counts, browser details, pagination attempts, stop reason, and limitations."),
|
|
3247
3240
|
inputSchema: InstagramProfileContentInputSchema,
|
|
3248
3241
|
outputSchema: InstagramProfileContentOutputSchema,
|
|
3249
3242
|
annotations: liveWebToolAnnotations("Instagram Profile Content Discovery")
|
|
3250
3243
|
}, async (input) => formatInstagramProfileContent(await executor2.instagramProfileContent(input), input));
|
|
3251
3244
|
server2.registerTool("instagram_media_download", {
|
|
3252
3245
|
title: "Instagram Post/Reel Media Download",
|
|
3253
|
-
description: withReportNote("Extract and download media from one Instagram post, reel, or tv URL. Use after instagram_profile_content or when the user gives a specific Instagram URL and wants the image, caption/text, reel audio/video tracks, optional muxed MP4, or optional transcript. Reels commonly expose separate video-only and audio-only MP4 tracks; this tool selects the best video and audio tracks and attempts muxing when ffmpeg is available.
|
|
3246
|
+
description: withReportNote("Extract and download media from one Instagram post, reel, or tv URL. Use after instagram_profile_content or when the user gives a specific Instagram URL and wants the image, caption/text, reel audio/video tracks, optional muxed MP4, or optional transcript. Reels commonly expose separate video-only and audio-only MP4 tracks; this tool selects the best video and audio tracks and attempts muxing when ffmpeg is available."),
|
|
3254
3247
|
inputSchema: InstagramMediaDownloadInputSchema,
|
|
3255
3248
|
outputSchema: InstagramMediaDownloadOutputSchema,
|
|
3256
3249
|
annotations: liveWebToolAnnotations("Instagram Post/Reel Media Download")
|