browserx 1.0.0 → 2.0.0

data/bot_categories.yml

@@ -0,0 +1,1522 @@
+---
+academic_research:
+- a patent crawler
+- academicbotrtu
+- aranea
+- autoconfig test from ustc
+- blackboard
+- bubing
+- cdscbot
+- cispa webcrawler
+- citeseerxbot
+- citoid
+- classla-web
+- easybib autocite
+- fedineko
+- gaisbot
+- gulper web bot
+- hgfalphaxcrawl/
+- icc-crawler
+- integromedb
+- internetmeasurement
+- irlbot
+- macocu
+- mithril-crawler
+- researchbot
+- researchscan.comsys.rwth-aachen.de
+- semanticscholarbot
+- spiderling
+- thesis-research-bot
+- thinklab
+- turnitin
+- turnitinbot
+- ugaresearchagent
+- vu-server-health-scanner/
+- web measure/
+- zoterotranslationserver/wmf
+advertising_and_marketing:
+- ";owler"
+- a360-search
+- adbeat.com
+- adbeat_bot
+- adkerneltopiccrawler
+- admantx
+- adsbot-google
+- adsbot-iab
+- adscanner
+- adstxtcrawler
+- advbot
+- amazing-searchbot
+- amazonadbot
+- audigentadbot
+- bidswitchbot
+- centro ads.txt crawler
+- channable
+- clickagy
+- contextad bot
+- contextualbot
+- criteobot
+- cxensebot
+- dvbot
+- ecovadissustainabilitybot
+- ezlynx
+- ezoicbot
+- facebookcatalog/
+- google-ads-conversions
+- google-adwords-express
+- google-adwords-instant
+- grapeshotcrawler
+- groovinaadsbot/
+- ias crawler
+- impact.com agent
+- kargobot-artemis
+- mediapartners-google
+- mediavinemetadataparser/
+- meta-externalads/
+- mgidbot
+- mixrankbot
+- moatbot
+- nanointeractive
+- netseer crawler
+- netshelter contentscan
+- newsroom.bi
+- nostocrawlerbot
+- outbrain
+- outclicksbot
+- peer39_crawler
+- pixalate.com
+- productsup.io/crawler
+- proximic
+- pulsepoint
+- quantcastbot/
+- qwarrybot
+- scope3
+- scraping@nytimes.com
+- smartologybot/
+- springservebot
+- tracking-quality-spider
+- ttd-content
+- uipbot
+- verity/
+- voluumdsp-content-bot
+- webgains-bot
+- weborama-fetcher
+- webspidermount
+- wknd-bot
+- yahoo ad monitoring
+- yahoo-ad-monitoring
+ai_agent:
+- agi agent
+- aranet-searchbot
+- cartai agentic commerce
+- chatgpt agent
+- claude-code
+- claude-web
+- crawl4ai
+- grok bot
+- harkbot
+- iaskbot
+- iaskspider
+- kimi-agent
+- kunatocrawler
+- novaact
+- opencode
+- payhawk invoice fetching agent
+- qodercli
+- reflectionbot
+ai_assistant:
+- amazon-qbusiness
+- amzn-user
+- bigsur.ai
+- chatgpt-user
+- claude-user
+- cohere-ai
+- duckassistbot
+- elevenlabsbot
+- exauser
+- geisthaus-pagefetcher
+- gemini-deep-research
+- googleagent-urlcontext
+- henkbot
+- kagi-fetcher
+- kimi-user
+- klaviyoaibot
+- linerbot
+- meta-externalfetcher
+- mistralai-user
+- notebooklm
+- perplexity-user
+- phindbot
+- poggio-citations
+- qualifiedbot
+- shap-user
+- yiyanbot
+ai_crawler:
+- agenttimes
+- ai2bot
+- aisearchbot
+- anthropic-ai
+- apifybot
+- apifywebsitecontentcrawler
+- awario
+- bedrockbot
+- bixelbot
+- brightbot
+- bytespider
+- chatglm-spider
+- claudebot
+- cloudvertexbot
+- cragcrawler
+- crawlspace
+- datenbank crawler
+- diffbot
+- doubaobot
+- echobot bot
+- echoboxbot
+- elevenlabscrawler
+- erniebot
+- extecontextcrawl
+- facebookbot
+- firecrawlagent
+- flyriverbot
+- friendlycrawler
+- google-firebase
+- gptbot
+- hunyuan
+- imagemind
+- imagespider
+- img2dataset
+- isonai-user
+- isscyberriskcrawler
+- kangaroo bot
+- kimibot
+- laion-huggingface-processor
+- linkreader/
+- meta-externalagent
+- moonshotbot
+- mozilla-tabstack
+- mycentralaiscraperbot
+- nagetbot
+- netestate imprint crawler
+- newsai
+- novellum
+- oai-adsbot
+- perplexityuser
+- quillbot
+- qwenbot
+- sbintuitionsbot
+- scrybot
+- semantic-visions
+- spider.com
+- taragroup intelligent bot
+- the knowledge ai
+- thinkbot
+- tiktokspider
+- tsm-turingos
+- wardbot
+- wpbot
+- xai-grok
+- yibot
+ai_search:
+- aiwebindex
+- amazon-kendra
+- amzn-searchbot
+- andibot
+- anomura
+- atlassian-bot
+- azureai-searchbot
+- ccbot
+- channel3bot
+- claude-searchbot
+- commoncrawl
+- exasearchbot
+- google-extended
+- grokappandroid
+- grokbot
+- keenablebot
+- kimi-searchbot
+- laiondownloader
+- linkupbot
+- mistralai-index
+- oai-searchbot
+- perplexitybot
+- poseidon research crawler
+- querit-searchbot
+- queritbot
+- shapbot
+- tavilybot
+- terra cotta
+- terracotta
+archiver:
+- archive-it
+- archive.org_bot
+- archivebot
+- archivebox
+- archiveteam archivebot
+- arquivo-web-crawler
+- arquivobot
+- authory
+- bibliotheque nacional de france crawler
+- bl.uk_lddc_bot
+- bl.uk_ldfc_bot
+- bne.es_bot
+- bnf.fr_bot
+- bnfbot
+- cc metadata scaper
+- cyotekwebcopy
+- europarchive.org
+- heritrix
+- ia_archiver
+- kaikki.org-digital-archive
+- kb.dk_bot
+- lac_iaharvester
+- library of congress web archiving
+- linkace/
+- linkarchiver
+- loc.gov/programs/web-archiving
+- miniature.io/
+- mirrorweb.com
+- mirrorwebcrawler
+- nlnz_iaharvester
+- nlux_iaharvester
+- offline explorer
+- sitesucker
+- smarshbot
+- trove
+- wayback
+- web-archive-net.com.bot
+- webcapture
+- webcopier
+- webzip
+- xy-archive-compliance
+browser_automation:
+- amazon-bedrock-agentcore-browser
+- amazonbuyforme
+- anchor browser
+- fluid
+- ghost inspector
+- google-agent
+- google-gemini-cli
+- googleagent-mariner
+- headlesschrome
+- lightpanda
+- manus-user
+- opencode-smartfetch
+- phantomjs
+- playwright
+- puppeteer
+- selenium
+- splash version/
+- testlocally/
+- trae/
+- twinagent
+feed_fetcher:
+- allafrica
+- artemis web reader
+- atom feed robot
+- bazqux
+- bestchange
+- blingerp
+- bloglines
+- blogtrottr
+- buttondown
+- capitalonebot
+- collapsify
+- commafeed
+- cozi-icalendar-feedreader
+- dlvr.it
+- everyfeed-spider
+- fedoraplanet
+- feed-fetcher
+- feedbin
+- feedbot
+- feedburner
+- feeder /
+- feeder.co
+- feedfetcher-google
+- feedflow
+- feedly
+- feedpresso content index bot
+- feedsearch bot
+- feedsearch-crawler
+- feedspot
+- feedwind
+- fever
+- flipboardrss
+- freshrss
+- g2reader-bot
+- innologica
+- inoreader
+- magpierss
+- mail.ru
+- mailchimp
+- miniflux/
+- monitorss/
+- naver blog rssbot
+- netvibes
+- newsblur
+- nicheindex
+- openrss
+- overcast/
+- page2rss
+- pocketcasts/
+- podchaserparser
+- podimo
+- protopage/
+- qcbot
+- rss reader
+- rss-is-dead.lol web bot
+- rss-parser
+- rss.social
+- rss2tg
+- rssapi/
+- rssbot/
+- rssingbot
+- rssmicro
+- rssreaderbot
+- shortpixel
+- simplepie
+- sottopopnone
+- specificfeeds
+- superfeedr
+- theoldreader.com
+- tiny tiny rss
+- upday/
+- yandorirssbot
+- yelpspider
+http_library:
+- aiohttp
+- alittle client
+- amazon cloudfront
+- anyevent
+- apache-httpclient
+- apachebench
+- asynchttpclient
+- axios
+- btwebclient
+- coldfusion
+- colly
+- crawler4j
+- curl
+- dap/nethttp
+- download ninja
+- eventmachine httpclient
+- go 1.1 package http
+- go http package
+- go-http-client
+- goparserbot
+- htmlparser
+- http_get
+- http_request2
+- httpclient
+- httpie
+- httpunit
+- httpurlconnection
+- httpx
+- httrack
+- jaunt
+- jetty
+- larbin
+- libwww-perl
+- lwp-trivial
+- mechanize
+- newspaper/
+- node-fetch
+- okhttp
+- pcore-http
+- phpcrawl
+- pycurl
+- python-httplib2
+- python-opengraph
+- python-requests
+- python-urllib
+- rawweb-bot
+- salesforce.com
+- scrapy
+- siege
+- simplecrawler
+- sindresorhus/got
+- trafilatura
+- typhoeus
+- wget
+monitoring_and_analytics:
+- abevalbot
+- accessible web bot
+- accessstatus
+- acquia optimize
+- activecomply
+- alertsite
+- appinsights
+- appsignal
+- atvowbot
+- automaton
+- better uptime bot
+- betteruptimebot
+- binarycanary
+- blackbox exporter/
+- bling erp
+- blockaid
+- blogvault
+- buddybot
+- bugsnag
+- capsulechecker
+- carbon-umbrella-bot
+- catchpoint
+- caveman-hunter
+- changedetection
+- check_http
+- checkhost
+- checkly
+- checkmarknetwork/
+- chirp
+- chrome-lighthouse
+- cloudflare
+- cludo
+- corporama matcher
+- crazy egg
+- critical css bot
+- cron-job.org
+- cronless
+- curebot
+- dareboost
+- dark visitor
+- datadog agent
+- datadog synthetic
+- datadogsynthetics
+- deepnoc
+- determ
+- dmbrowser
+- dominicbot
+- dotcom-monitor
+- dow jones searchbot
+- downnotifier.com
+- downtimedetector/
+- drataautopilot
+- dreamhost data team
+- drupalbot
+- dubbotbot
+- easycron/
+- easydns monitoring
+- edgewatch
+- elb-healthchecker
+- elmah.io uptime monitoring
+- elmahio-uptimebot
+- emoney advisor
+- evouptimebot
+- experibot
+- fastdast
+- fedidb
+- fedilist agent
+- fedistatscrawler
+- fedreporter bot for ffiec
+- feedvalidator
+- freshbot
+- freshpingbot
+- genomecrawlerd
+- google page speed insights
+- google trust services
+- googlestackdrivermonitoring
+- googlestackdrivermonitoring-uptimechecks
+- gotosocial
+- gotsitemonitor.com
+- gtmetrix
+- halobot
+- hanaleibot
+- headonlyscraper
+- hetrixtools
+- honeybadger uptime check
+- honeybadgerbot
+- hosttracker/
+- hydrozen.io
+- infosifter
+- internetvista monitor
+- irokez.cz monitoring
+- isdownbot/
+- jetmon/
+- keycdn-tools/
+- known agent
+- linko
+- linkpadbot
+- loadtimebot
+- logicmonitor
+- loopimprovements.com
+- magnetmebot
+- magus bot
+- mediatoolkitbot
+- minicrawler
+- minoru's fediverse crawler
+- modatscanner
+- monitis -
+- monitoring360bot/
+- monsidobot
+- monspark/
+- montastic-monitor
+- montasticmonitor
+- mxtoolbox
+- netumo
+- netvigie
+- new york times newsgathering
+- newrelicmonitor
+- newrelicpinger
+- newrelicsynthetics
+- newsify feed fetcher
+- nixstatsbot
+- noahbot
+- nodeping
+- noibu
+- observepoint
+- ohdear/
+- onetrust
+- onlineornot.com_bot
+- owler-w
+- paessler
+- pagespeed/
+- panopta
+- parselysharesbot
+- pingadmin.ru/
+- pingdom
+- pingping.io/
+- plagawarebot
+- poduptime
+- pomothy-bot
+- prtg network monitor
+- prtgcloudbot/
+- pulsetic.com
+- qualys
+- rackspace monitoring/
+- rakuten image extraction bot
+- rigor)
+- ruxitsynthetic
+- scouturlmonitor
+- sentinel-linkcheck
+- sentry/
+- sequelwp
+- server density service monitoring
+- serverdensity
+- silktide
+- sindup
+- site24x7
+- siteimprove.com
+- sitelock
+- siteuptime
+- sleepbot
+- solarwinds/
+- spectate/
+- splunk
+- sqwatcher
+- statuscake
+- streamline3bot/
+- sv-watchagent
+- synthesibot
+- synthetic-monitoring-agent/
+- termly
+- thousandeyes
+- tsmbot
+- updown
+- uptime-kuma
+- uptime/
+- uptimebot.org
+- uptimebot/
+- uptimedoctor
+- uptimerobot
+- uptimestatistics
+- uptimia
+- uptrends
+- urlsuma.de crawler
+- validator.nu
+- videootv bot
+- vigil
+- vmcrawl
+- w3c-checklink
+- w3c-mobileok
+- w3c_css_validator
+- w3c_i18n-checker
+- w3c_unicorn
+- w3c_validator
+- wasalive-bot
+- watchbot monitoring robot
+- watchforbot
+- watchful
+- watchmouse
+- webmon
+- websiteops
+- websitepulse
+- wedos online
+- wheresitup.com/
+- wisebot
+- wormly
+- wovn crawler
+- wpmu dev hub
+- wpmudev uptime monitor
+- wpumbrella
+- www.uptime.com
+- ylt chrome
+- zabbix
+page_preview:
+- aasa-bot
+- addthis
+- arenaunfurlbot
+- awe.sm
+- bingpreview
+- bluesky-domain-status-classifier
+- bluesky/
+- blueskypreviewbot
+- bufferbot
+- bufferlinkpreviewbot
+- chatwork linkpreview
+- clickuplinkunfurler
+- dazzle bluesky bot
+- dingtalkbot-linkservice
+- discordbot
+- discourse forum onebox
+- embedly
+- evernoterichlinkbot
+- everyonesocialbot
+- facebookexternalhit
+- facebot
+- fastmailua
+- flipboardproxy
+- friendica
+- gg peekbot
+- github-camo
+- goodreads
+- google web preview
+- googleimageproxy
+- groupmebot
+- hatena
+- hatena::bookmark
+- iframely
+- jakarta commons
+- jetslide
+- keybasebot
+- lemmy
+- linespider
+- linkedinbot
+- listgembot
+- longurl
+- loomlybot
+- mastodon
+- mattermost-bot/
+- meta-externalhit/
+- meta-webindexer
+- metaiab facebook
+- microsoftpreview/
+- newsharecounts
+- notion/
+- odklbot/
+- opengraph.io/
+- orlo-linkpreview/
+- pinterest
+- quicklook
+- quora-bot
+- redditbot
+- shortlinktranslate
+- shortwave image fetcher
+- skypeuripreview
+- slack
+- slack-imgproxy
+- slack-linkexpanding
+- slackbot
+- snap url preview service
+- snapchat
+- snapurlpreview/
+- summalybot
+- supabase paired crawler
+- synapse
+- telegrambot
+- tumblr/
+- tweetmemebot
+- twilioproxy/
+- twitterbot
+- valve/steam
+- vkrobot
+- vkshare
+- whatsapp
+- xing bot
+- yahoo link preview
+- yahoomailproxy
+scanner:
+- abuse.xmco.fr
+- acunetix
+- akismetbot
+- aliyunsecbot
+- antibot
+- artemisbot
+- asnriskscorer
+- bad neighborhood header detector
+- binlar
+- bitdiscovery
+- bitsightbot/
+- blackduck-fd
+- browserbot-observer
+- censysinspect/
+- cert pl
+- certchief
+- cookie-maestro
+- cookiebot
+- cookiehub bot
+- cookiehubscan
+- cookiehubverify
+- cookieyesbot
+- csirt.cz
+- cyberpatrol
+- detectify
+- digicert dcv
+- dirbuster
+- dnsscanner
+- do not track verifier
+- ec2linkfinder
+- emailwolf
+- expanse
+- feed image audit
+- filterdb.iss.net/crawler
+- foregenix
+- google-trust-services/
+- hardenize
+- hstspreload-bot
+- http banner detection
+- ip-web-crawler.com
+- iubenda-radar/
+- jugendschutzprogramm-crawler
+- k7mlwcbot
+- keydrop.io
+- krawlerbot
+- l9explore
+- l9scan
+- legalmonster
+- let's encrypt
+- lightspeedsystemscrawler
+- ltx71
+- masscan
+- mrgbot/
+- nessus
+- netcraft
+- netsystemsresearch
+- nikto
+- ning
+- ninja bot
+- nmap scripting engine
+- nstoolsbot/
+- openvas
+- probelyspdr/
+- projectshield-urlcheck
+- reverseengineeringbot
+- riddler
+- safednsbot
+- sansec security monitor/
+- sbl-bot
+- securityheaders
+- software-security-research
+- sqlmap
+- ssl labs
+- sucuri
+- test certificate info
+- tls tester
+- twinwavescanner
+- unshorten.it!
+- urlbeebot
+- ut-dorkbot
+- vaultpress
+- verispider
+- virustotal
+- visionheight.com/scan
+- wanscannerbot
+- watchtowr
+- whatweb/
+- wpscan
+- wpsec/
+- wtotem
+- zgrab
+search_engine_crawler:
+- 360spider
+- 7siters
+- a6-indexer
+- aboundex
+- acoonbot
+- addsearchbot
+- adidxbot
+- aihitbot
+- alexa
+- algolia crawler
+- amazonbot
+- amazonproductdiscovery
+- amazonsellerinitiatedlisting
+- apis-google
+- appengine-google
+- applebot
+- ask jeeves
+- aspiegelbot
+- baidu
+- biglotron
+- bingbot
+- blekkobot
+- bot.araturka.com
+- br-crawler
+- bravebot
+- cliqzbot
+- coccoc
+- convera
+- coveobot
+- daumoa
+- deusu/
+- developers.google.com/+/web/snippet
+- developers.google.com/+/web/snippet/
+- discobot
+- duckduckbot
+- duckduckgo-favicons-bot
+- edisterbot
+- exabot
+- fast enterprise crawler
+- fast-webcrawler
+- fediindex
+- femtosearchbot
+- findfiles.net
+- findthatfile
+- findxbot
+- fiperbot
+- flamingo_searchengine
+- fleebsbot
+- fluffy
+- fr-crawler
+- freespoke/
+- funnelback
+- fyndsearchengine-crawler
+- fyndsearchengine-recrawler
+- geedobot
+- geedoproductsearch
+- geedoshopproductfinder
+- gigablast
+- gigabot
+- globalwebsearch
+- gluten free crawler/
+- golfe/
+- google favicon
+- google-certificates-bridge
+- google-inspectiontool
+- google-physicalweb
+- google-read-aloud
+- google-safety
+- google-site-verification
+- google-structured-data-testing-tool
+- google-xrawler
+- googlebot
+- googleother
+- googleproducer
+- greppr web crawler
+- grub.org
+- gsa-crawler
+- https://developers.google.com/+/web/snippet
+- iboubot
+- ichiro
+- imagesiftbot
+- indeedbot
+- indeedjobbot
+- inetdex-bot
+- intelx.io_bot
+- isonsearchbot
+- istellabot
+- jobboersebot
+- jobswithgptcom-bot
+- jooble
+- jyxobot
+- kagibot/
+- lb-spider
+- level9searchbot/
+- linguee bot
+- lyonl-asset-proxy
+- lyonl-crawler
+- magibot
+- mail.ru_bot
+- matchorysearch
+- mojeekbot
+- msnbot
+- msnbot-media
+- msrbot
+- napbot
+- naverbot
+- nestdaddybot
+- nicecrawler
+- ntentbot
+- nutch
+- officestorebot
+- oicrawler
+- openhosebot
+- openindexspider
+- opentheboxbot
+- orangebot
+- owler (ows.eu
+- pangubot
+- petalbot
+- phxbot
+- playstore-google
+- psbot
+- psbot-page
+- purebot
+- qwantbot
+- qwantify
+- safesearch microdata crawler
+- scribdbot
+- scrubby
+- search.marginalia.nu
+- searcherweb
+- searcherxweb
+- searchexpress
+- seekbot
+- seekport
+- seznambot
+- seznamhomepagecrawler/
+- sitesearch360/
+- slurp
+- smtnetpmbot
+- sogou
+- sosospider
+- startpageprivateimageproxy/
+- stepstonecrawlbot
+- storebot-google
+- stractbot
+- swiftbot
+- swisscows
+- tagoobot
+- teoma
+- theinternetsearchx
+- timpibot
+- top100.rambler.ru crawler
+- toutiaospider
+- trovitbot
+- umbot
+- usinenouvellecrawler
+- voilabot
+- vsusearchspider/
+- wadoobot
+- wbsearchbot
+- wepchsearchengine
+- wesee
+- wikido
+- wocbot
+- wordupinfosearch
+- woriobot
+- wotbox
+- yacybot
+- yandex
+- yanga
+- yeti
+- yisouspider
+- youbot
+- youdaobot
+- zealbot
+- zoominfo-
+- zyborg
+search_engine_optimization:
+- 007ac9 crawler
+- 2ip bot
+- 2ip.ru
+- abot
+- acapbot
+- adagiobot
+- adminlabs
+- advanced_crawler
+- adventurer
+- agakidsbot
+- agencyanalyticsbot
+- ahrefsbot
+- ahrefssiteaudit
+- alienfarm
+- allorigins
+- anderspink
+- apercite
+- appsiteassociation
+- aragog/
+- asana/
+- attracta
+- audisto crawler
+- audistobot
+- awesomecrawler
+- b2b bot
+- backlinkcrawler
+- barkrowler
+- bdbot/
+- bdcbot
+- bdfetch
+- beeperbot
+- behloolbot
+- bitbot
+- bitbucket-webhooks
+- bitlybot
+- blexbot
+- blp_bbot
+- boardgamepricesbot
+- bot-pge.chlooe.com
+- botify
+- botpoke
+- boxcarbot
+- brainobot
+- brandonbot
+- brandverity
+- brandwatch
+- brightedge crawler
+- brokenlinkcheck.com
+- bublupbot
+- bushbaby
+- butterfly
+- buzzbot
+- caliberbot
+- caliperbot
+- capitaloneshopping
+- careerbot
+- centcombot/
+- centuryb.o.t9
+- centurybot
+- certytags
+- chargebeebot
+- charlotte
+- chrome privacy preserving prefetch proxy
+- cincraw
+- cipacrawler
+- cirrusexplorer
+- claritybot/
+- clearscopebot
+- cloudtrellis
+- cmschecker
+- cocolyzebot
+- cognitiveseo bot
+- cognitiveseo.com
+- collection@infegy.com
+- companybook-crawler
+- content crawler spider
+- contentkingapp
+- contxbot
+- convermax
+- cotoyogi
+- crawler_eb_germany
+- crawlson
+- crawlyprojectcrawler
+- criticalcss.com
+- crunchbot
+- crusty/
+- crystalsemanticsbot
+- cutbot
+- cxk_bot
+- daspeedbot
+- datafeedwatch
+- dataforseobot
+- datagnionbot
+- datanyze
+- dataprovider.com
+- dcrawl
+- dead link checker
+- deadlinkchecker
+- deepcrawl
+- deskyobot
+- df bot
+- digg deeper
+- digincore bot
+- disqus
+- dmbot
+- dnbcrawler
+- dnyzbot
+- domain re-animator bot
+- domaincrawler
+- domains project/
+- domainsbot
+- domainstatsbot
+- domcopbot
+- dotbot
+- dragonbot
+- drupact
+- e.ventures investment crawler
+- easybill-importmanager
+- edansbot
+- electricmonk
+- emoneybot
+- epicbot
+- epivozcrawler
+- erepublik.tools
+- eright
+- ev-crawler/
+- evc-batch
+- exodusmovement
+- extlinksbot
+- ezgif
+- ezooms
+- factset_spyderbot
+- fdl stats bot
+- fetch/
+- fidget-spinner-bot
+- finditanswersbot
+- findlink
+- firmograph
+- friendly testing bot
+- friendly-spider
+- fuelbot
+- fullstorybot/
+- fuseonbot/
+- fyrebot
+- g00g1e.net
+- g2 web services
+- garlik
+- gdnplus.com
+- genieo
+- getthit.com
+- giftedvisitor scan
+- gingercrawler
+- gnam gnam spider
+- gnowitnewsbot
+- goodzer
+- google-apps-script
+- google-area120
+- googleassociationservice
+- googledocs
+- gopay
+- gowikibot
+- grammarly/
+- gregcrawler
+- grobbot
+- grouphigh/
+- grover/
+- gslfbot
+- guestpostsbot/
+- happywing
+- harsilbot/
+- hawaiibot
+- hcardvalidator
+- hello world
+- helloworkjobpostingbot/
+- hifibot/
+- hlidam.to robot
+- hotjar
+- hubspot
+- hype machine/
+- hypestat
+- hyscore.io
+- ibm crawler
+- icbot/
+- idg/eu
+- ifttt/
+- imagefetcher/
+- imgproxy/
+- impendoom-bot/
+- imrbot
+- indy library
+- infoobot
+- innguma/
+- instapaper
+- integromat/
+- intelium_bot
+- interfaxscanbot
+- ioncrawl
+- ips-agent
+- iskanie
+- it2media-domain-crawler
+- james bot
+- jamie's spider
+- java
+- jpg-newsbot
+- jumio
+- kangaroobot/
+- kazbtbot/
+- kemvibot
+- keys-so-bot
+- kinsta-bot
+- klaviyo/
+- komodiabot
+- konturbot/
+- kosmiobot
+- kstandbot
+- kukei.eu-bot/
+- landau-media-spider
+- laserlikebot
+- lastmodbot/
+- leikibot
+- linkcheckerbot
+- linkdex
+- linkdexbot
+- linkisbot
+- linksindexerbot/
+- linktiger
+- linkwalker/
+- lipperhey
+- livelapbot
+- lssbot
+- lssrocketcrawler
+- luminator-robots
+- macrobondbot
+- madbbot/
+- magellan
+- magicsearchdev/
+- magnet.me-web/
+- magpie-crawler
+- mainwp/
+- managewp
+- mappydata
+- marketgoo/
+- marketingminer
+- mauibot
+- mavifinds
+- mb-linkchecker
+- mbcrawler
+- mediamonitoringbot/
+- megaindex
+- megaindex.ru
+- meltwaternews
+- memorybot
+- mergadobot
+- metainspector
+- metajobbot
+- metauri
+- metorik
+- mindupbot
+- missinglettrbot/
+- mixnodecache/
+- mj12bot
+- mlbot
+- modernizebot
+- modularconnector/
+- mollie http client
+- monibot
+- montools.com
+- moodlebot
+- moreover
+- motominerbot/
+- mtrobot
+- muckrack
+- multiviewbot
+- my-tiny-bot
+- mybot/
+- nbertaupete95
+- nerdbynature.bot
+- nerdybot
+- netapi
+- netestate ne crawler
+- neticle crawler
+- netpeakcheckerbot/
+- netresearchserver
+- newsnow/
+- niki-bot
+- nimbostratus-bot
+- nitro-
+- nitrobot
+- nomore404.com robot
+- noorobot
+- nooshub/
+- nuzzel
+- nvdorz
+- nyt_scraping
+- omgili
+- omnisend/
+- oncrawl/
+- online domain tools
+- opengraphcheck
+- operator/
+- optimizer)
+- orbbot/
+- outsellurlvalidator
+- page-preview-tool
+- pagepeeker/
+- pagething
+- pandalytics
+- panscient
+- paperlibot
+- paqlebot/
+- parse.ly scraper/
+- paypal/
+- pdf24 url to pdf
+- penthouse critical path css generator
+- piplbot
+- plesk screenshot bot
+- pocketparser
+- postrank
+- potions/
+- pr-cy.ru
+- pressenginebot
+- pricedroneshoppingbot/
+- priecobot/
+- primalbot
+- printfriendly.com
+- privacyawarebot
+- pro-sitemaps/
+- ps_daily/
+- pwabuilderhttpagent
+- pywikibot/
+- rankactivelinkbot
+- rankurbot/
+- ravencrawler/
+- re-animator
+- readable/
+- reelevant/
+- refindbot
+- regionstuttgartbot
+- remove.bg/
+- repolookoutbot
+- retool/
+- retrevopageanalyzer
+- retrolistecom/
+- revvimgort/
+- reward-gateway
+- ridderbot
+- rivva
+- robotschecker/
+- rogerbot
+- rsiteauditor
+- ruby
+- rytebot
+- s4a-probe-bot/
+- sandobacrawler/
+- schema-markup-validator
+- scoop.it/
+- scoutjet
+- scrapeherobot/
+- screaming frog seo spider
+- screeenly-bot
+- searchatlas
+- sebot-wa
+- seewithkids
+- semaltbot/
+- semanticbot
+- sempi.tech
+- semrushbot
+- sendgrid event api
+- sentibot
+- senutobot
+- seo-audit
+- seo4ajax.com
+- seobilitybot
+- seokicks
+- seolizer
+- seolyt
+- seoscanners
+- seostar.co
+- seoulbot
+- serankingbacklinksbot
+- serendeputybot
+- serpstatbot/
+- serptimizerbot
+- serverhunterspider/
+- sfdc-callout/
+- shopify-captain-hook
+- sidetrade indexer bot
+- simplescraper
+- sincerasyntheticuser/
+- sistrix
+- siteauditbot/
+- sitebot
+- sitebulb
+- sitecheck
+- siteexplorer.info
+- sitescorebot
+- sitesoverpagesbot
+- skroutz imagebot
+- skroutzbot
+- skyworkspider
+- slaccalebot
+- slickbot/
+- smtbot
+- snacktory
+- snipcart/
+- socialrank
+- sora pos/
+- sparkpost
+- sparkshipping
+- spbot
+- ssssbot/
+- stape/
+- startmebot/
+- statistikaustria/
+- statsdronebot
+- statusnestbacklinkspider
+- storygizebot
+- stripe/
+- summify
+- surdotlybot
+- surveybot
+- sustainabilitycrawler
+- svix-webhooks/
+- swifteqlinkchecker
+- swimgbot
+- sysomos
+- t3versionsbot
+- tactiscout/
+- tangibleebot
+- tchelebi/
+- test-bot
+- testcrawler
+- testuri
+- textrazor
+- tigerbot
+- tineye
+- tombapublicwebcrawler
+- toplistbot
+- traackr
+- tracemyfile
+- trellis-services
+- trendictionbot
+- trendsmap
+- trentwil.es
+- trustly/
+- tweakers
+- tweetedtimes
+- twingly
+- twurly
+- uaslinkchecker/
+- upflow
+- urlappendbot
+- urlcheckr/
+- useebookchecker/
+- vagabondo
+- vebidoobot
+- velenpublicwebcrawler
+- veoozbot
+- vercel-screenshot
+- vercelbot
+- videootvbot
+- vuhuvbot
+- web screen service by hyperhost
+- webceo
+- webcompanycrawler
+- webdatastats
+- webflow
+- webprosbot/
+- website-info.net-robot
+- websniffer/
+- webwikibot/
+- webz.io
+- webzio
+- wellknownbot
+- whatsmyip.org
+- wheregoes.com
+- wirereaderbot/
+- wjhro/
+- wmtips.com/
+- woobot
+- woorank
+- wordcountbot/
+- wordpress
+- wordup-1
+- workona-favicon-service/
+- worldbot
+- wovncrawler/
+- wowlink crawler/
+- wp time capsule
+- wpmu dev broken link checker
+- wrtnbot
+- xenu link sleuth
+- xml sitemaps generator
+- xovibot
+- xovionpagecrawler
+- yahoocachesystem
+- yellowbrandprotectionbot
+- yextbot/
+- yokoygroupag/
+- yoozbot
+- yuuperbot
+- zapier
+- zebra-v2-bot
+- zenback bot
+- zendesk webhook
+- zombiebot/
+- zoombot
+- zoominfobot
+- zuperlistbot/
+- zzhbot
+security:
+- comodo ssl checker
+- comodo-webinspector-crawler