RubyGems - browserx - Versions diffs - 2.0.0 → 2.1.0 - Mend

browserx 2.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (26) hide show

checksums.yaml +4 -4
data/CHANGELOG.md +19 -1
data/README.md +14 -1
data/data/bot_categories.json +1 -0
data/data/bot_exceptions.json +1 -0
data/data/bot_ip_ranges.json +1 -0
data/data/bots.json +1 -0
data/data/languages.json +1 -0
data/data/samsung.json +1 -0
data/data/search_engines.json +1 -0
data/lib/browser/accept_language.rb +1 -1
data/lib/browser/bot/ip_ranges.rb +1 -1
data/lib/browser/bot.rb +10 -4
data/lib/browser/browser.rb +17 -1
data/lib/browser/device/samsung.rb +1 -1
data/lib/browser/rails.rb +1 -0
data/lib/browser/testing.rb +2 -0
data/lib/browser/version.rb +1 -1
metadata +8 -8
data/bot_categories.yml +0 -1522
data/bot_exceptions.yml +0 -5
data/bot_ip_ranges.yml +0 -3035
data/bots.yml +0 -1576
data/languages.yml +0 -113
data/samsung.yml +0 -138
data/search_engines.yml +0 -7

data/bot_categories.yml DELETED Viewed

@@ -1,1522 +0,0 @@
----
-academic_research:
-- a patent crawler
-- academicbotrtu
-- aranea
-- autoconfig test from ustc
-- blackboard
-- bubing
-- cdscbot
-- cispa webcrawler
-- citeseerxbot
-- citoid
-- classla-web
-- easybib autocite
-- fedineko
-- gaisbot
-- gulper web bot
-- hgfalphaxcrawl/
-- icc-crawler
-- integromedb
-- internetmeasurement
-- irlbot
-- macocu
-- mithril-crawler
-- researchbot
-- researchscan.comsys.rwth-aachen.de
-- semanticscholarbot
-- spiderling
-- thesis-research-bot
-- thinklab
-- turnitin
-- turnitinbot
-- ugaresearchagent
-- vu-server-health-scanner/
-- web measure/
-- zoterotranslationserver/wmf
-advertising_and_marketing:
-- ";owler"
-- a360-search
-- adbeat.com
-- adbeat_bot
-- adkerneltopiccrawler
-- admantx
-- adsbot-google
-- adsbot-iab
-- adscanner
-- adstxtcrawler
-- advbot
-- amazing-searchbot
-- amazonadbot
-- audigentadbot
-- bidswitchbot
-- centro ads.txt crawler
-- channable
-- clickagy
-- contextad bot
-- contextualbot
-- criteobot
-- cxensebot
-- dvbot
-- ecovadissustainabilitybot
-- ezlynx
-- ezoicbot
-- facebookcatalog/
-- google-ads-conversions
-- google-adwords-express
-- google-adwords-instant
-- grapeshotcrawler
-- groovinaadsbot/
-- ias crawler
-- impact.com agent
-- kargobot-artemis
-- mediapartners-google
-- mediavinemetadataparser/
-- meta-externalads/
-- mgidbot
-- mixrankbot
-- moatbot
-- nanointeractive
-- netseer crawler
-- netshelter contentscan
-- newsroom.bi
-- nostocrawlerbot
-- outbrain
-- outclicksbot
-- peer39_crawler
-- pixalate.com
-- productsup.io/crawler
-- proximic
-- pulsepoint
-- quantcastbot/
-- qwarrybot
-- scope3
-- scraping@nytimes.com
-- smartologybot/
-- springservebot
-- tracking-quality-spider
-- ttd-content
-- uipbot
-- verity/
-- voluumdsp-content-bot
-- webgains-bot
-- weborama-fetcher
-- webspidermount
-- wknd-bot
-- yahoo ad monitoring
-- yahoo-ad-monitoring
-ai_agent:
-- agi agent
-- aranet-searchbot
-- cartai agentic commerce
-- chatgpt agent
-- claude-code
-- claude-web
-- crawl4ai
-- grok bot
-- harkbot
-- iaskbot
-- iaskspider
-- kimi-agent
-- kunatocrawler
-- novaact
-- opencode
-- payhawk invoice fetching agent
-- qodercli
-- reflectionbot
-ai_assistant:
-- amazon-qbusiness
-- amzn-user
-- bigsur.ai
-- chatgpt-user
-- claude-user
-- cohere-ai
-- duckassistbot
-- elevenlabsbot
-- exauser
-- geisthaus-pagefetcher
-- gemini-deep-research
-- googleagent-urlcontext
-- henkbot
-- kagi-fetcher
-- kimi-user
-- klaviyoaibot
-- linerbot
-- meta-externalfetcher
-- mistralai-user
-- notebooklm
-- perplexity-user
-- phindbot
-- poggio-citations
-- qualifiedbot
-- shap-user
-- yiyanbot
-ai_crawler:
-- agenttimes
-- ai2bot
-- aisearchbot
-- anthropic-ai
-- apifybot
-- apifywebsitecontentcrawler
-- awario
-- bedrockbot
-- bixelbot
-- brightbot
-- bytespider
-- chatglm-spider
-- claudebot
-- cloudvertexbot
-- cragcrawler
-- crawlspace
-- datenbank crawler
-- diffbot
-- doubaobot
-- echobot bot
-- echoboxbot
-- elevenlabscrawler
-- erniebot
-- extecontextcrawl
-- facebookbot
-- firecrawlagent
-- flyriverbot
-- friendlycrawler
-- google-firebase
-- gptbot
-- hunyuan
-- imagemind
-- imagespider
-- img2dataset
-- isonai-user
-- isscyberriskcrawler
-- kangaroo bot
-- kimibot
-- laion-huggingface-processor
-- linkreader/
-- meta-externalagent
-- moonshotbot
-- mozilla-tabstack
-- mycentralaiscraperbot
-- nagetbot
-- netestate imprint crawler
-- newsai
-- novellum
-- oai-adsbot
-- perplexityuser
-- quillbot
-- qwenbot
-- sbintuitionsbot
-- scrybot
-- semantic-visions
-- spider.com
-- taragroup intelligent bot
-- the knowledge ai
-- thinkbot
-- tiktokspider
-- tsm-turingos
-- wardbot
-- wpbot
-- xai-grok
-- yibot
-ai_search:
-- aiwebindex
-- amazon-kendra
-- amzn-searchbot
-- andibot
-- anomura
-- atlassian-bot
-- azureai-searchbot
-- ccbot
-- channel3bot
-- claude-searchbot
-- commoncrawl
-- exasearchbot
-- google-extended
-- grokappandroid
-- grokbot
-- keenablebot
-- kimi-searchbot
-- laiondownloader
-- linkupbot
-- mistralai-index
-- oai-searchbot
-- perplexitybot
-- poseidon research crawler
-- querit-searchbot
-- queritbot
-- shapbot
-- tavilybot
-- terra cotta
-- terracotta
-archiver:
-- archive-it
-- archive.org_bot
-- archivebot
-- archivebox
-- archiveteam archivebot
-- arquivo-web-crawler
-- arquivobot
-- authory
-- bibliotheque nacional de france crawler
-- bl.uk_lddc_bot
-- bl.uk_ldfc_bot
-- bne.es_bot
-- bnf.fr_bot
-- bnfbot
-- cc metadata scaper
-- cyotekwebcopy
-- europarchive.org
-- heritrix
-- ia_archiver
-- kaikki.org-digital-archive
-- kb.dk_bot
-- lac_iaharvester
-- library of congress web archiving
-- linkace/
-- linkarchiver
-- loc.gov/programs/web-archiving
-- miniature.io/
-- mirrorweb.com
-- mirrorwebcrawler
-- nlnz_iaharvester
-- nlux_iaharvester
-- offline explorer
-- sitesucker
-- smarshbot
-- trove
-- wayback
-- web-archive-net.com.bot
-- webcapture
-- webcopier
-- webzip
-- xy-archive-compliance
-browser_automation:
-- amazon-bedrock-agentcore-browser
-- amazonbuyforme
-- anchor browser
-- fluid
-- ghost inspector
-- google-agent
-- google-gemini-cli
-- googleagent-mariner
-- headlesschrome
-- lightpanda
-- manus-user
-- opencode-smartfetch
-- phantomjs
-- playwright
-- puppeteer
-- selenium
-- splash version/
-- testlocally/
-- trae/
-- twinagent
-feed_fetcher:
-- allafrica
-- artemis web reader
-- atom feed robot
-- bazqux
-- bestchange
-- blingerp
-- bloglines
-- blogtrottr
-- buttondown
-- capitalonebot
-- collapsify
-- commafeed
-- cozi-icalendar-feedreader
-- dlvr.it
-- everyfeed-spider
-- fedoraplanet
-- feed-fetcher
-- feedbin
-- feedbot
-- feedburner
-- feeder /
-- feeder.co
-- feedfetcher-google
-- feedflow
-- feedly
-- feedpresso content index bot
-- feedsearch bot
-- feedsearch-crawler
-- feedspot
-- feedwind
-- fever
-- flipboardrss
-- freshrss
-- g2reader-bot
-- innologica
-- inoreader
-- magpierss
-- mail.ru
-- mailchimp
-- miniflux/
-- monitorss/
-- naver blog rssbot
-- netvibes
-- newsblur
-- nicheindex
-- openrss
-- overcast/
-- page2rss
-- pocketcasts/
-- podchaserparser
-- podimo
-- protopage/
-- qcbot
-- rss reader
-- rss-is-dead.lol web bot
-- rss-parser
-- rss.social
-- rss2tg
-- rssapi/
-- rssbot/
-- rssingbot
-- rssmicro
-- rssreaderbot
-- shortpixel
-- simplepie
-- sottopopnone
-- specificfeeds
-- superfeedr
-- theoldreader.com
-- tiny tiny rss
-- upday/
-- yandorirssbot
-- yelpspider
-http_library:
-- aiohttp
-- alittle client
-- amazon cloudfront
-- anyevent
-- apache-httpclient
-- apachebench
-- asynchttpclient
-- axios
-- btwebclient
-- coldfusion
-- colly
-- crawler4j
-- curl
-- dap/nethttp
-- download ninja
-- eventmachine httpclient
-- go 1.1 package http
-- go http package
-- go-http-client
-- goparserbot
-- htmlparser
-- http_get
-- http_request2
-- httpclient
-- httpie
-- httpunit
-- httpurlconnection
-- httpx
-- httrack
-- jaunt
-- jetty
-- larbin
-- libwww-perl
-- lwp-trivial
-- mechanize
-- newspaper/
-- node-fetch
-- okhttp
-- pcore-http
-- phpcrawl
-- pycurl
-- python-httplib2
-- python-opengraph
-- python-requests
-- python-urllib
-- rawweb-bot
-- salesforce.com
-- scrapy
-- siege
-- simplecrawler
-- sindresorhus/got
-- trafilatura
-- typhoeus
-- wget
-monitoring_and_analytics:
-- abevalbot
-- accessible web bot
-- accessstatus
-- acquia optimize
-- activecomply
-- alertsite
-- appinsights
-- appsignal
-- atvowbot
-- automaton
-- better uptime bot
-- betteruptimebot
-- binarycanary
-- blackbox exporter/
-- bling erp
-- blockaid
-- blogvault
-- buddybot
-- bugsnag
-- capsulechecker
-- carbon-umbrella-bot
-- catchpoint
-- caveman-hunter
-- changedetection
-- check_http
-- checkhost
-- checkly
-- checkmarknetwork/
-- chirp
-- chrome-lighthouse
-- cloudflare
-- cludo
-- corporama matcher
-- crazy egg
-- critical css bot
-- cron-job.org
-- cronless
-- curebot
-- dareboost
-- dark visitor
-- datadog agent
-- datadog synthetic
-- datadogsynthetics
-- deepnoc
-- determ
-- dmbrowser
-- dominicbot
-- dotcom-monitor
-- dow jones searchbot
-- downnotifier.com
-- downtimedetector/
-- drataautopilot
-- dreamhost data team
-- drupalbot
-- dubbotbot
-- easycron/
-- easydns monitoring
-- edgewatch
-- elb-healthchecker
-- elmah.io uptime monitoring
-- elmahio-uptimebot
-- emoney advisor
-- evouptimebot
-- experibot
-- fastdast
-- fedidb
-- fedilist agent
-- fedistatscrawler
-- fedreporter bot for ffiec
-- feedvalidator
-- freshbot
-- freshpingbot
-- genomecrawlerd
-- google page speed insights
-- google trust services
-- googlestackdrivermonitoring
-- googlestackdrivermonitoring-uptimechecks
-- gotosocial
-- gotsitemonitor.com
-- gtmetrix
-- halobot
-- hanaleibot
-- headonlyscraper
-- hetrixtools
-- honeybadger uptime check
-- honeybadgerbot
-- hosttracker/
-- hydrozen.io
-- infosifter
-- internetvista monitor
-- irokez.cz monitoring
-- isdownbot/
-- jetmon/
-- keycdn-tools/
-- known agent
-- linko
-- linkpadbot
-- loadtimebot
-- logicmonitor
-- loopimprovements.com
-- magnetmebot
-- magus bot
-- mediatoolkitbot
-- minicrawler
-- minoru's fediverse crawler
-- modatscanner
-- monitis -
-- monitoring360bot/
-- monsidobot
-- monspark/
-- montastic-monitor
-- montasticmonitor
-- mxtoolbox
-- netumo
-- netvigie
-- new york times newsgathering
-- newrelicmonitor
-- newrelicpinger
-- newrelicsynthetics
-- newsify feed fetcher
-- nixstatsbot
-- noahbot
-- nodeping
-- noibu
-- observepoint
-- ohdear/
-- onetrust
-- onlineornot.com_bot
-- owler-w
-- paessler
-- pagespeed/
-- panopta
-- parselysharesbot
-- pingadmin.ru/
-- pingdom
-- pingping.io/
-- plagawarebot
-- poduptime
-- pomothy-bot
-- prtg network monitor
-- prtgcloudbot/
-- pulsetic.com
-- qualys
-- rackspace monitoring/
-- rakuten image extraction bot
-- rigor)
-- ruxitsynthetic
-- scouturlmonitor
-- sentinel-linkcheck
-- sentry/
-- sequelwp
-- server density service monitoring
-- serverdensity
-- silktide
-- sindup
-- site24x7
-- siteimprove.com
-- sitelock
-- siteuptime
-- sleepbot
-- solarwinds/
-- spectate/
-- splunk
-- sqwatcher
-- statuscake
-- streamline3bot/
-- sv-watchagent
-- synthesibot
-- synthetic-monitoring-agent/
-- termly
-- thousandeyes
-- tsmbot
-- updown
-- uptime-kuma
-- uptime/
-- uptimebot.org
-- uptimebot/
-- uptimedoctor
-- uptimerobot
-- uptimestatistics
-- uptimia
-- uptrends
-- urlsuma.de crawler
-- validator.nu
-- videootv bot
-- vigil
-- vmcrawl
-- w3c-checklink
-- w3c-mobileok
-- w3c_css_validator
-- w3c_i18n-checker
-- w3c_unicorn
-- w3c_validator
-- wasalive-bot
-- watchbot monitoring robot
-- watchforbot
-- watchful
-- watchmouse
-- webmon
-- websiteops
-- websitepulse
-- wedos online
-- wheresitup.com/
-- wisebot
-- wormly
-- wovn crawler
-- wpmu dev hub
-- wpmudev uptime monitor
-- wpumbrella
-- www.uptime.com
-- ylt chrome
-- zabbix
-page_preview:
-- aasa-bot
-- addthis
-- arenaunfurlbot
-- awe.sm
-- bingpreview
-- bluesky-domain-status-classifier
-- bluesky/
-- blueskypreviewbot
-- bufferbot
-- bufferlinkpreviewbot
-- chatwork linkpreview
-- clickuplinkunfurler
-- dazzle bluesky bot
-- dingtalkbot-linkservice
-- discordbot
-- discourse forum onebox
-- embedly
-- evernoterichlinkbot
-- everyonesocialbot
-- facebookexternalhit
-- facebot
-- fastmailua
-- flipboardproxy
-- friendica
-- gg peekbot
-- github-camo
-- goodreads
-- google web preview
-- googleimageproxy
-- groupmebot
-- hatena
-- hatena::bookmark
-- iframely
-- jakarta commons
-- jetslide
-- keybasebot
-- lemmy
-- linespider
-- linkedinbot
-- listgembot
-- longurl
-- loomlybot
-- mastodon
-- mattermost-bot/
-- meta-externalhit/
-- meta-webindexer
-- metaiab facebook
-- microsoftpreview/
-- newsharecounts
-- notion/
-- odklbot/
-- opengraph.io/
-- orlo-linkpreview/
-- pinterest
-- quicklook
-- quora-bot
-- redditbot
-- shortlinktranslate
-- shortwave image fetcher
-- skypeuripreview
-- slack
-- slack-imgproxy
-- slack-linkexpanding
-- slackbot
-- snap url preview service
-- snapchat
-- snapurlpreview/
-- summalybot
-- supabase paired crawler
-- synapse
-- telegrambot
-- tumblr/
-- tweetmemebot
-- twilioproxy/
-- twitterbot
-- valve/steam
-- vkrobot
-- vkshare
-- whatsapp
-- xing bot
-- yahoo link preview
-- yahoomailproxy
-scanner:
-- abuse.xmco.fr
-- acunetix
-- akismetbot
-- aliyunsecbot
-- antibot
-- artemisbot
-- asnriskscorer
-- bad neighborhood header detector
-- binlar
-- bitdiscovery
-- bitsightbot/
-- blackduck-fd
-- browserbot-observer
-- censysinspect/
-- cert pl
-- certchief
-- cookie-maestro
-- cookiebot
-- cookiehub bot
-- cookiehubscan
-- cookiehubverify
-- cookieyesbot
-- csirt.cz
-- cyberpatrol
-- detectify
-- digicert dcv
-- dirbuster
-- dnsscanner
-- do not track verifier
-- ec2linkfinder
-- emailwolf
-- expanse
-- feed image audit
-- filterdb.iss.net/crawler
-- foregenix
-- google-trust-services/
-- hardenize
-- hstspreload-bot
-- http banner detection
-- ip-web-crawler.com
-- iubenda-radar/
-- jugendschutzprogramm-crawler
-- k7mlwcbot
-- keydrop.io
-- krawlerbot
-- l9explore
-- l9scan
-- legalmonster
-- let's encrypt
-- lightspeedsystemscrawler
-- ltx71
-- masscan
-- mrgbot/
-- nessus
-- netcraft
-- netsystemsresearch
-- nikto
-- ning
-- ninja bot
-- nmap scripting engine
-- nstoolsbot/
-- openvas
-- probelyspdr/
-- projectshield-urlcheck
-- reverseengineeringbot
-- riddler
-- safednsbot
-- sansec security monitor/
-- sbl-bot
-- securityheaders
-- software-security-research
-- sqlmap
-- ssl labs
-- sucuri
-- test certificate info
-- tls tester
-- twinwavescanner
-- unshorten.it!
-- urlbeebot
-- ut-dorkbot
-- vaultpress
-- verispider
-- virustotal
-- visionheight.com/scan
-- wanscannerbot
-- watchtowr
-- whatweb/
-- wpscan
-- wpsec/
-- wtotem
-- zgrab
-search_engine_crawler:
-- 360spider
-- 7siters
-- a6-indexer
-- aboundex
-- acoonbot
-- addsearchbot
-- adidxbot
-- aihitbot
-- alexa
-- algolia crawler
-- amazonbot
-- amazonproductdiscovery
-- amazonsellerinitiatedlisting
-- apis-google
-- appengine-google
-- applebot
-- ask jeeves
-- aspiegelbot
-- baidu
-- biglotron
-- bingbot
-- blekkobot
-- bot.araturka.com
-- br-crawler
-- bravebot
-- cliqzbot
-- coccoc
-- convera
-- coveobot
-- daumoa
-- deusu/
-- developers.google.com/+/web/snippet
-- developers.google.com/+/web/snippet/
-- discobot
-- duckduckbot
-- duckduckgo-favicons-bot
-- edisterbot
-- exabot
-- fast enterprise crawler
-- fast-webcrawler
-- fediindex
-- femtosearchbot
-- findfiles.net
-- findthatfile
-- findxbot
-- fiperbot
-- flamingo_searchengine
-- fleebsbot
-- fluffy
-- fr-crawler
-- freespoke/
-- funnelback
-- fyndsearchengine-crawler
-- fyndsearchengine-recrawler
-- geedobot
-- geedoproductsearch
-- geedoshopproductfinder
-- gigablast
-- gigabot
-- globalwebsearch
-- gluten free crawler/
-- golfe/
-- google favicon
-- google-certificates-bridge
-- google-inspectiontool
-- google-physicalweb
-- google-read-aloud
-- google-safety
-- google-site-verification
-- google-structured-data-testing-tool
-- google-xrawler
-- googlebot
-- googleother
-- googleproducer
-- greppr web crawler
-- grub.org
-- gsa-crawler
-- https://developers.google.com/+/web/snippet
-- iboubot
-- ichiro
-- imagesiftbot
-- indeedbot
-- indeedjobbot
-- inetdex-bot
-- intelx.io_bot
-- isonsearchbot
-- istellabot
-- jobboersebot
-- jobswithgptcom-bot
-- jooble
-- jyxobot
-- kagibot/
-- lb-spider
-- level9searchbot/
-- linguee bot
-- lyonl-asset-proxy
-- lyonl-crawler
-- magibot
-- mail.ru_bot
-- matchorysearch
-- mojeekbot
-- msnbot
-- msnbot-media
-- msrbot
-- napbot
-- naverbot
-- nestdaddybot
-- nicecrawler
-- ntentbot
-- nutch
-- officestorebot
-- oicrawler
-- openhosebot
-- openindexspider
-- opentheboxbot
-- orangebot
-- owler (ows.eu
-- pangubot
-- petalbot
-- phxbot
-- playstore-google
-- psbot
-- psbot-page
-- purebot
-- qwantbot
-- qwantify
-- safesearch microdata crawler
-- scribdbot
-- scrubby
-- search.marginalia.nu
-- searcherweb
-- searcherxweb
-- searchexpress
-- seekbot
-- seekport
-- seznambot
-- seznamhomepagecrawler/
-- sitesearch360/
-- slurp
-- smtnetpmbot
-- sogou
-- sosospider
-- startpageprivateimageproxy/
-- stepstonecrawlbot
-- storebot-google
-- stractbot
-- swiftbot
-- swisscows
-- tagoobot
-- teoma
-- theinternetsearchx
-- timpibot
-- top100.rambler.ru crawler
-- toutiaospider
-- trovitbot
-- umbot
-- usinenouvellecrawler
-- voilabot
-- vsusearchspider/
-- wadoobot
-- wbsearchbot
-- wepchsearchengine
-- wesee
-- wikido
-- wocbot
-- wordupinfosearch
-- woriobot
-- wotbox
-- yacybot
-- yandex
-- yanga
-- yeti
-- yisouspider
-- youbot
-- youdaobot
-- zealbot
-- zoominfo-
-- zyborg
-search_engine_optimization:
-- 007ac9 crawler
-- 2ip bot
-- 2ip.ru
-- abot
-- acapbot
-- adagiobot
-- adminlabs
-- advanced_crawler
-- adventurer
-- agakidsbot
-- agencyanalyticsbot
-- ahrefsbot
-- ahrefssiteaudit
-- alienfarm
-- allorigins
-- anderspink
-- apercite
-- appsiteassociation
-- aragog/
-- asana/
-- attracta
-- audisto crawler
-- audistobot
-- awesomecrawler
-- b2b bot
-- backlinkcrawler
-- barkrowler
-- bdbot/
-- bdcbot
-- bdfetch
-- beeperbot
-- behloolbot
-- bitbot
-- bitbucket-webhooks
-- bitlybot
-- blexbot
-- blp_bbot
-- boardgamepricesbot
-- bot-pge.chlooe.com
-- botify
-- botpoke
-- boxcarbot
-- brainobot
-- brandonbot
-- brandverity
-- brandwatch
-- brightedge crawler
-- brokenlinkcheck.com
-- bublupbot
-- bushbaby
-- butterfly
-- buzzbot
-- caliberbot
-- caliperbot
-- capitaloneshopping
-- careerbot
-- centcombot/
-- centuryb.o.t9
-- centurybot
-- certytags
-- chargebeebot
-- charlotte
-- chrome privacy preserving prefetch proxy
-- cincraw
-- cipacrawler
-- cirrusexplorer
-- claritybot/
-- clearscopebot
-- cloudtrellis
-- cmschecker
-- cocolyzebot
-- cognitiveseo bot
-- cognitiveseo.com
-- collection@infegy.com
-- companybook-crawler
-- content crawler spider
-- contentkingapp
-- contxbot
-- convermax
-- cotoyogi
-- crawler_eb_germany
-- crawlson
-- crawlyprojectcrawler
-- criticalcss.com
-- crunchbot
-- crusty/
-- crystalsemanticsbot
-- cutbot
-- cxk_bot
-- daspeedbot
-- datafeedwatch
-- dataforseobot
-- datagnionbot
-- datanyze
-- dataprovider.com
-- dcrawl
-- dead link checker
-- deadlinkchecker
-- deepcrawl
-- deskyobot
-- df bot
-- digg deeper
-- digincore bot
-- disqus
-- dmbot
-- dnbcrawler
-- dnyzbot
-- domain re-animator bot
-- domaincrawler
-- domains project/
-- domainsbot
-- domainstatsbot
-- domcopbot
-- dotbot
-- dragonbot
-- drupact
-- e.ventures investment crawler
-- easybill-importmanager
-- edansbot
-- electricmonk
-- emoneybot
-- epicbot
-- epivozcrawler
-- erepublik.tools
-- eright
-- ev-crawler/
-- evc-batch
-- exodusmovement
-- extlinksbot
-- ezgif
-- ezooms
-- factset_spyderbot
-- fdl stats bot
-- fetch/
-- fidget-spinner-bot
-- finditanswersbot
-- findlink
-- firmograph
-- friendly testing bot
-- friendly-spider
-- fuelbot
-- fullstorybot/
-- fuseonbot/
-- fyrebot
-- g00g1e.net
-- g2 web services
-- garlik
-- gdnplus.com
-- genieo
-- getthit.com
-- giftedvisitor scan
-- gingercrawler
-- gnam gnam spider
-- gnowitnewsbot
-- goodzer
-- google-apps-script
-- google-area120
-- googleassociationservice
-- googledocs
-- gopay
-- gowikibot
-- grammarly/
-- gregcrawler
-- grobbot
-- grouphigh/
-- grover/
-- gslfbot
-- guestpostsbot/
-- happywing
-- harsilbot/
-- hawaiibot
-- hcardvalidator
-- hello world
-- helloworkjobpostingbot/
-- hifibot/
-- hlidam.to robot
-- hotjar
-- hubspot
-- hype machine/
-- hypestat
-- hyscore.io
-- ibm crawler
-- icbot/
-- idg/eu
-- ifttt/
-- imagefetcher/
-- imgproxy/
-- impendoom-bot/
-- imrbot
-- indy library
-- infoobot
-- innguma/
-- instapaper
-- integromat/
-- intelium_bot
-- interfaxscanbot
-- ioncrawl
-- ips-agent
-- iskanie
-- it2media-domain-crawler
-- james bot
-- jamie's spider
-- java
-- jpg-newsbot
-- jumio
-- kangaroobot/
-- kazbtbot/
-- kemvibot
-- keys-so-bot
-- kinsta-bot
-- klaviyo/
-- komodiabot
-- konturbot/
-- kosmiobot
-- kstandbot
-- kukei.eu-bot/
-- landau-media-spider
-- laserlikebot
-- lastmodbot/
-- leikibot
-- linkcheckerbot
-- linkdex
-- linkdexbot
-- linkisbot
-- linksindexerbot/
-- linktiger
-- linkwalker/
-- lipperhey
-- livelapbot
-- lssbot
-- lssrocketcrawler
-- luminator-robots
-- macrobondbot
-- madbbot/
-- magellan
-- magicsearchdev/
-- magnet.me-web/
-- magpie-crawler
-- mainwp/
-- managewp
-- mappydata
-- marketgoo/
-- marketingminer
-- mauibot
-- mavifinds
-- mb-linkchecker
-- mbcrawler
-- mediamonitoringbot/
-- megaindex
-- megaindex.ru
-- meltwaternews
-- memorybot
-- mergadobot
-- metainspector
-- metajobbot
-- metauri
-- metorik
-- mindupbot
-- missinglettrbot/
-- mixnodecache/
-- mj12bot
-- mlbot
-- modernizebot
-- modularconnector/
-- mollie http client
-- monibot
-- montools.com
-- moodlebot
-- moreover
-- motominerbot/
-- mtrobot
-- muckrack
-- multiviewbot
-- my-tiny-bot
-- mybot/
-- nbertaupete95
-- nerdbynature.bot
-- nerdybot
-- netapi
-- netestate ne crawler
-- neticle crawler
-- netpeakcheckerbot/
-- netresearchserver
-- newsnow/
-- niki-bot
-- nimbostratus-bot
-- nitro-
-- nitrobot
-- nomore404.com robot
-- noorobot
-- nooshub/
-- nuzzel
-- nvdorz
-- nyt_scraping
-- omgili
-- omnisend/
-- oncrawl/
-- online domain tools
-- opengraphcheck
-- operator/
-- optimizer)
-- orbbot/
-- outsellurlvalidator
-- page-preview-tool
-- pagepeeker/
-- pagething
-- pandalytics
-- panscient
-- paperlibot
-- paqlebot/
-- parse.ly scraper/
-- paypal/
-- pdf24 url to pdf
-- penthouse critical path css generator
-- piplbot
-- plesk screenshot bot
-- pocketparser
-- postrank
-- potions/
-- pr-cy.ru
-- pressenginebot
-- pricedroneshoppingbot/
-- priecobot/
-- primalbot
-- printfriendly.com
-- privacyawarebot
-- pro-sitemaps/
-- ps_daily/
-- pwabuilderhttpagent
-- pywikibot/
-- rankactivelinkbot
-- rankurbot/
-- ravencrawler/
-- re-animator
-- readable/
-- reelevant/
-- refindbot
-- regionstuttgartbot
-- remove.bg/
-- repolookoutbot
-- retool/
-- retrevopageanalyzer
-- retrolistecom/
-- revvimgort/
-- reward-gateway
-- ridderbot
-- rivva
-- robotschecker/
-- rogerbot
-- rsiteauditor
-- ruby
-- rytebot
-- s4a-probe-bot/
-- sandobacrawler/
-- schema-markup-validator
-- scoop.it/
-- scoutjet
-- scrapeherobot/
-- screaming frog seo spider
-- screeenly-bot
-- searchatlas
-- sebot-wa
-- seewithkids
-- semaltbot/
-- semanticbot
-- sempi.tech
-- semrushbot
-- sendgrid event api
-- sentibot
-- senutobot
-- seo-audit
-- seo4ajax.com
-- seobilitybot
-- seokicks
-- seolizer
-- seolyt
-- seoscanners
-- seostar.co
-- seoulbot
-- serankingbacklinksbot
-- serendeputybot
-- serpstatbot/
-- serptimizerbot
-- serverhunterspider/
-- sfdc-callout/
-- shopify-captain-hook
-- sidetrade indexer bot
-- simplescraper
-- sincerasyntheticuser/
-- sistrix
-- siteauditbot/
-- sitebot
-- sitebulb
-- sitecheck
-- siteexplorer.info
-- sitescorebot
-- sitesoverpagesbot
-- skroutz imagebot
-- skroutzbot
-- skyworkspider
-- slaccalebot
-- slickbot/
-- smtbot
-- snacktory
-- snipcart/
-- socialrank
-- sora pos/
-- sparkpost
-- sparkshipping
-- spbot
-- ssssbot/
-- stape/
-- startmebot/
-- statistikaustria/
-- statsdronebot
-- statusnestbacklinkspider
-- storygizebot
-- stripe/
-- summify
-- surdotlybot
-- surveybot
-- sustainabilitycrawler
-- svix-webhooks/
-- swifteqlinkchecker
-- swimgbot
-- sysomos
-- t3versionsbot
-- tactiscout/
-- tangibleebot
-- tchelebi/
-- test-bot
-- testcrawler
-- testuri
-- textrazor
-- tigerbot
-- tineye
-- tombapublicwebcrawler
-- toplistbot
-- traackr
-- tracemyfile
-- trellis-services
-- trendictionbot
-- trendsmap
-- trentwil.es
-- trustly/
-- tweakers
-- tweetedtimes
-- twingly
-- twurly
-- uaslinkchecker/
-- upflow
-- urlappendbot
-- urlcheckr/
-- useebookchecker/
-- vagabondo
-- vebidoobot
-- velenpublicwebcrawler
-- veoozbot
-- vercel-screenshot
-- vercelbot
-- videootvbot
-- vuhuvbot
-- web screen service by hyperhost
-- webceo
-- webcompanycrawler
-- webdatastats
-- webflow
-- webprosbot/
-- website-info.net-robot
-- websniffer/
-- webwikibot/
-- webz.io
-- webzio
-- wellknownbot
-- whatsmyip.org
-- wheregoes.com
-- wirereaderbot/
-- wjhro/
-- wmtips.com/
-- woobot
-- woorank
-- wordcountbot/
-- wordpress
-- wordup-1
-- workona-favicon-service/
-- worldbot
-- wovncrawler/
-- wowlink crawler/
-- wp time capsule
-- wpmu dev broken link checker
-- wrtnbot
-- xenu link sleuth
-- xml sitemaps generator
-- xovibot
-- xovionpagecrawler
-- yahoocachesystem
-- yellowbrandprotectionbot
-- yextbot/
-- yokoygroupag/
-- yoozbot
-- yuuperbot
-- zapier
-- zebra-v2-bot
-- zenback bot
-- zendesk webhook
-- zombiebot/
-- zoombot
-- zoominfobot
-- zuperlistbot/
-- zzhbot
-security:
-- comodo ssl checker
-- comodo-webinspector-crawler