browserx 1.0.0 → 2.0.0

data/bots.yml

@@ -325,3 +325,1252 @@ java: Generic Java http library
 libwww-perl: Perl client-server library
 lwp-trivial: Another Perl library
 ruby: Ruby
+
+# Synced from crawler-user-agents and ai.robots.txt by script/sync_bots.rb: edit above this line.
+007ac9 crawler: 007ac9 Crawler
+2ip bot: 2ip bot
+2ip.ru: 2ip.ru
+7siters: 7Siters
+";owler": ";Owler"
+a patent crawler: A Patent Crawler
+a360-search: A360-Search
+a6-indexer: A6-Indexer
+aasa-bot: AASA-Bot
+abevalbot: ABEvalBot
+aboundex: Aboundex
+abuse.xmco.fr: abuse.xmco.fr
+academicbotrtu: AcademicBotRTU
+acapbot: acapbot
+accessible web bot: Accessible Web Bot
+accessstatus: AccessStatus
+acoonbot: acoonbot
+acquia optimize: Acquia optimize
+activecomply: ActiveComply
+acunetix: acunetix
+adagiobot: AdagioBot
+adbeat.com: adbeat.com
+adbeat_bot: adbeat_bot
+addsearchbot: AddSearchBot
+adidxbot: Adidxbot
+adkerneltopiccrawler: AdkernelTopicCrawler
+adminlabs: AdminLabs
+adsbot-iab: AdsBot-IAB
+adscanner: adscanner
+advanced_crawler: advanced_crawler
+adventurer: Adventurer
+agakidsbot: AGAKIDSBOT
+agencyanalyticsbot: AgencyAnalyticsBot
+agenttimes: AgentTimes
+agi agent: AGI Agent
+ahrefssiteaudit: AhrefsSiteAudit
+ai2bot: AI2Bot
+aihitbot: aiHitBot
+aiohttp: aiohttp
+aisearchbot: AISearchBot
+aiwebindex: AIWebIndex
+akismetbot: AkismetBot
+alertsite: AlertSite
+algolia crawler: Algolia Crawler
+alienfarm: alienfarm
+alittle client: ALittle Client
+aliyunsecbot: AliyunSecBot
+allafrica: AllAfrica
+allorigins: allOrigins
+amazing-searchbot: Amazing-SearchBot
+amazon cloudfront: Amazon CloudFront
+amazon-kendra: amazon-kendra
+amazon-qbusiness: amazon-QBusiness
+amazonadbot: AmazonAdBot
+amazonproductdiscovery: AmazonProductDiscovery
+amazonsellerinitiatedlisting: AmazonSellerInitiatedListing
+amzn-searchbot: Amzn-SearchBot
+amzn-user: Amzn-User
+andibot: Andibot
+anomura: Anomura
+antibot: antibot
+anyevent: AnyEvent
+apercite: Apercite
+apifybot: ApifyBot
+apifywebsitecontentcrawler: ApifyWebsiteContentCrawler
+appinsights: AppInsights
+appsiteassociation: AppSiteAssociation
+aragog/: Aragog
+aranea: Aranea
+aranet-searchbot: Aranet-SearchBot
+archive-it: Archive-It
+archivebot: ArchiveBot
+archivebox: ArchiveBox
+arenaunfurlbot: ArenaUnfurlBot
+arquivo-web-crawler: Arquivo-web-crawler
+arquivobot: ArquivoBot
+artemis web reader: artemis web reader
+artemisbot: ArtemisBot
+asana/: Asana
+asnriskscorer: asnriskscorer
+aspiegelbot: AspiegelBot
+atlassian-bot: atlassian-bot
+atom feed robot: Atom Feed Robot
+attracta: Attracta
+atvowbot: AtVowBot
+audigentadbot: AudigentAdBot
+audisto crawler: Audisto Crawler
+audistobot: AudistoBot
+authory: Authory
+autoconfig test from ustc: Autoconfig Test from USTC
+automaton: Automaton
+awario: Awario
+awesomecrawler: awesomecrawler
+axios: axios
+azureai-searchbot: AzureAI-SearchBot
+b2b bot: B2B Bot
+backlinkcrawler: backlinkcrawler
+bad neighborhood header detector: Bad Neighborhood Header Detector
+bazqux: BazQux
+bdbot/: BDBot
+bdfetch: BDFetch
+bedrockbot: bedrockbot
+beeperbot: BeeperBot
+behloolbot: BehloolBot
+bestchange: BestChange
+better uptime bot: Better Uptime Bot
+betteruptimebot: BetterUptimeBot
+bibliotheque nacional de france crawler: Bibliotheque Nacional de France Crawler
+bidswitchbot: bidswitchbot
+biglotron: BIGLOTRON
+bigsur.ai: bigsur.ai
+binarycanary: BinaryCanary
+binlar: binlar
+bitbot: BitBot
+bitbucket-webhooks: Bitbucket-Webhooks
+bitdiscovery: bitdiscovery
+bitsightbot/: BitSightBot
+bixelbot: BixelBot
+bl.uk_lddc_bot: bl.uk_lddc_bot
+bl.uk_ldfc_bot: bl.uk_ldfc_bot
+blackboard: Blackboard
+blackbox exporter/: Blackbox Exporter
+blackduck-fd: BlackDuck-FD
+bling erp: Bling ERP
+blingerp: BlingERP
+blockaid: Blockaid
+bloglines: Bloglines
+blogtrottr: Blogtrottr
+blogvault: BlogVault
+blp_bbot: BLP_bbot
+bluesky-domain-status-classifier: bluesky-domain-status-classifier
+bluesky/: Bluesky
+blueskypreviewbot: BlueskyPreviewBot
+bne.es_bot: bne.es_bot
+bnf.fr_bot: bnf.fr_bot
+bnfbot: BnFBot
+boardgamepricesbot: BoardGamePricesBot
+bot-pge.chlooe.com: bot-pge.chlooe.com
+bot.araturka.com: Bot.AraTurka.com
+botify: botify
+botpoke: BotPoke
+boxcarbot: BoxcarBot
+br-crawler: br-crawler
+brainobot: brainobot
+brandonbot: BrandONbot
+brandverity: BrandVerity
+brandwatch: Brandwatch
+bravebot: Bravebot
+brightbot: Brightbot
+brightedge crawler: BrightEdge Crawler
+brokenlinkcheck.com: brokenlinkcheck.com
+browserbot-observer: BrowserBot-Observer
+btwebclient: BTWebClient
+bublupbot: BublupBot
+buddybot: BuddyBot
+bufferlinkpreviewbot: BufferLinkPreviewBot
+bugsnag: Bugsnag
+bushbaby: Bushbaby
+buttondown: Buttondown
+caliberbot: CaliberBot
+caliperbot: Caliperbot
+capitalonebot: CapitalOneBot
+capitaloneshopping: CapitalOneShopping
+capsulechecker: CapsuleChecker
+carbon-umbrella-bot: carbon-umbrella-bot
+careerbot: careerbot
+cartai agentic commerce: CartAI Agentic Commerce
+catchpoint: Catchpoint
+caveman-hunter: caveman-hunter
+cc metadata scaper: CC Metadata Scaper
+cdscbot: CDSCbot
+censysinspect/: CensysInspect
+centcombot/: CentComBot
+centro ads.txt crawler: Centro Ads.txt Crawler
+centuryb.o.t9: centuryb.o.t9
+centurybot: centurybot
+cert pl: CERT PL
+certchief: CertChief
+certytags: certytags
+changedetection: changedetection
+channable: channable
+channel3bot: Channel3Bot
+chargebeebot: ChargeBeeBot
+charlotte: Charlotte
+chatglm-spider: ChatGLM-Spider
+chatgpt agent: ChatGPT Agent
+chatwork linkpreview: Chatwork LinkPreview
+checkhost: CheckHost
+checkmarknetwork/: CheckMarkNetwork
+chirp: Chirp
+chrome privacy preserving prefetch proxy: Chrome Privacy Preserving Prefetch Proxy
+cincraw: Cincraw
+cirrusexplorer: CirrusExplorer
+cispa webcrawler: CISPA Webcrawler
+citeseerxbot: citeseerxbot
+citoid: Citoid
+claritybot/: ClarityBot
+classla-web: CLASSLA-web
+claude-code: Claude-Code
+claude-searchbot: Claude-SearchBot
+claude-user: Claude-User
+claudebot: ClaudeBot
+clearscopebot: Clearscopebot
+clickagy: Clickagy
+clickuplinkunfurler: ClickUpLinkUnfurler
+cloudtrellis: Cloudtrellis
+cloudvertexbot: CloudVertexBot
+cludo: cludo
+cmschecker: CMSChecker
+coccoc: coccoc
+cocolyzebot: Cocolyzebot
+cognitiveseo bot: cognitiveSEO Bot
+cognitiveseo.com: cognitiveseo.com
+collapsify: Collapsify
+collection@infegy.com: collection@infegy.com
+colly: colly
+commafeed: CommaFeed
+companybook-crawler: Companybook-Crawler
+content crawler spider: content crawler spider
+contentkingapp: contentkingapp
+contextad bot: ContextAd Bot
+contextualbot: ContextualBot
+contxbot: contxbot
+convera: convera
+convermax: Convermax
+cookie-maestro: cookie-maestro
+cookiebot: Cookiebot
+cookiehub bot: CookieHub Bot
+cookiehubscan: CookieHubScan
+cookiehubverify: CookieHubVerify
+cookieyesbot: CookieYesbot
+corporama matcher: Corporama matcher
+cotoyogi: Cotoyogi
+coveobot: Coveobot
+cozi-icalendar-feedreader: Cozi-iCalendar-FeedReader
+cragcrawler: CragCrawler
+crawl4ai: Crawl4AI
+crawler4j: crawler4j
+crawler_eb_germany: crawler_eb_germany
+crawlson: Crawlson
+crawlspace: Crawlspace
+crawlyprojectcrawler: CrawlyProjectCrawler
+crazy egg: Crazy Egg
+criteobot: CriteoBot
+critical css bot: Critical CSS Bot
+criticalcss.com: Criticalcss.com
+cron-job.org: cron-job.org
+cronless: Cronless
+crunchbot: CrunchBot
+crusty/: crusty
+crystalsemanticsbot: CrystalSemanticsBot
+csirt.cz: csirt.cz
+curebot: Curebot
+cutbot: Cutbot
+cxensebot: cXensebot
+cxk_bot: CXK_Bot
+cyberpatrol: CyberPatrol
+cyotekwebcopy: CyotekWebCopy
+dareboost: DareBoost
+dark visitor: Dark Visitor
+daspeedbot: DaspeedBot
+datadog agent: Datadog Agent
+datadog synthetic: Datadog Synthetic
+datadogsynthetics: DatadogSynthetics
+dataforseobot: DataForSeoBot
+dataprovider.com: Dataprovider.com
+datenbank crawler: Datenbank Crawler
+dazzle bluesky bot: Dazzle BlueSky Bot
+dcrawl: dcrawl
+dead link checker: Dead Link Checker
+deadlinkchecker: deadlinkchecker
+deepcrawl: DeepCrawl
+deepnoc: deepnoc
+deskyobot: Deskyobot
+detectify: Detectify
+determ: Determ
+deusu/: DeuSu
+developers.google.com/+/web/snippet: developers.google.com/+/web/snippet
+df bot: DF Bot
+digg deeper: Digg Deeper
+digicert dcv: DigiCert DCV
+digincore bot: Digincore bot
+dingtalkbot-linkservice: DingTalkBot-LinkService
+dirbuster: dirBuster
+discobot: discobot
+discordbot: Discordbot
+discourse forum onebox: Discourse Forum Onebox
+disqus: Disqus
+dlvr.it: dlvr.it
+dmbot: Dmbot
+dmbrowser: DMBrowser
+dnbcrawler: DnBCrawler
+dnsscanner: DNSScanner
+dnyzbot: DnyzBot
+do not track verifier: Do Not Track Verifier
+domaincrawler: domaincrawler
+domains project/: Domains Project
+domainstatsbot: DomainStatsBot
+domcopbot: DomCopBot
+dominicbot: DominicBot
+dotcom-monitor: Dotcom-Monitor
+doubaobot: DoubaoBot
+dow jones searchbot: Dow Jones Searchbot
+download ninja: Download Ninja
+downnotifier.com: downnotifier.com
+downtimedetector/: DowntimeDetector
+dragonbot: Dragonbot
+drataautopilot: DrataAutopilot
+dreamhost data team: DreamHost Data Team
+drupact: drupact
+drupalbot: Drupalbot
+dubbotbot: Dubbotbot
+duckassistbot: DuckAssistBot
+duckduckgo-favicons-bot: DuckDuckGo-Favicons-Bot
+dvbot: DVbot
+e.ventures investment crawler: e.ventures Investment Crawler
+easybib autocite: EasyBib AutoCite
+easybill-importmanager: easybill-ImportManager
+easycron/: EasyCron
+easydns monitoring: easyDNS Monitoring
+ec2linkfinder: ec2linkfinder
+echobot bot: Echobot Bot
+echoboxbot: EchoboxBot
+ecovadissustainabilitybot: EcoVadisSustainabilityBot
+edansbot: edansbot
+edgewatch: EdgeWatch
+edisterbot: edisterbot
+electricmonk: electricmonk
+elevenlabsbot: ElevenlabsBot
+elevenlabscrawler: ElevenlabsCrawler
+elmah.io uptime monitoring: elmah.io Uptime Monitoring
+elmahio-uptimebot: elmahio-uptimebot
+emailwolf: EmailWolf
+emoney advisor: eMoney Advisor
+emoneybot: eMoneyBot
+epicbot: epicbot
+epivozcrawler: EpivozCrawler
+erepublik.tools: eRepublik.tools
+eright: eright
+erniebot: ERNIEBot
+europarchive.org: europarchive.org
+ev-crawler/: ev-crawler
+evc-batch: evc-batch
+evernoterichlinkbot: EvernoteRichLinkBot
+everyfeed-spider: everyfeed-spider
+evouptimebot: EvoUptimeBot
+exasearchbot: ExaSearchBot
+exauser: ExaUser
+exodusmovement: ExodusMovement
+expanse: Expanse
+experibot: Experibot
+extecontextcrawl: ExteContextCrawl
+extlinksbot: ExtLinksBot
+ezgif: Ezgif
+ezlynx: EzLynx
+ezoicbot: EzoicBot
+ezooms: ezooms
+facebookcatalog/: facebookcatalog
+factset_spyderbot: Factset_spyderbot
+fast enterprise crawler: FAST Enterprise Crawler
+fast-webcrawler: FAST-WebCrawler
+fastdast: FastDAST
+fastmailua: FastmailUA
+fdl stats bot: FDL Stats Bot
+fedidb: FediDB
+fediindex: FediIndex
+fedilist agent: FediList Agent
+fedineko: Fedineko
+fedistatscrawler: fedistatsCrawler
+fedoraplanet: fedoraplanet
+fedreporter bot for ffiec: FedReporter Bot for FFIEC
+feed image audit: Feed Image Audit
+feed-fetcher: Feed-Fetcher
+feedbin: Feedbin
+feedbot: feedbot
+feeder /: 'Feeder '
+feeder.co: feeder.co
+feedflow: FeedFlow
+feedly: Feedly
+feedpresso content index bot: Feedpresso Content Index Bot
+feedsearch bot: Feedsearch Bot
+feedsearch-crawler: Feedsearch-Crawler
+feedspot: Feedspot
+feedvalidator: FeedValidator
+feedwind: Feedwind
+femtosearchbot: FemtosearchBot
+fetch/: Fetch
+fever: Fever
+fidget-spinner-bot: fidget-spinner-bot
+filterdb.iss.net/crawler: filterdb.iss.net/crawler
+findfiles.net: FindFiles.net
+finditanswersbot: FindITAnswersbot
+findlink: findlink
+findthatfile: findthatfile
+fiperbot: fiperbot
+firecrawlagent: FirecrawlAgent
+firmograph: FirmoGraph
+flamingo_searchengine: Flamingo_SearchEngine
+fleebsbot: FleebsBot
+flipboardrss: FlipboardRSS
+fluffy: fluffy
+flyriverbot: Flyriverbot
+foregenix: Foregenix
+fr-crawler: fr-crawler
+freespoke/: Freespoke
+freshbot: Freshbot
+freshpingbot: FreshpingBot
+freshrss: FreshRSS
+friendica: Friendica
+friendly testing bot: Friendly testing bot
+friendly-spider: friendly-spider
+friendlycrawler: FriendlyCrawler
+fuelbot: fuelbot
+fullstorybot/: FullStoryBot
+funnelback: Funnelback
+fuseonbot/: FuseonBot
+fyndsearchengine-crawler: FyndSearchEngine-Crawler
+fyndsearchengine-recrawler: FyndSearchEngine-ReCrawler
+g00g1e.net: g00g1e.net
+g2 web services: G2 Web Services
+g2reader-bot: g2reader-bot
+gaisbot: Gaisbot
+gdnplus.com: gdnplus.com
+geedobot: GeedoBot
+geedoproductsearch: GeedoProductSearch
+geedoshopproductfinder: GeedoShopProductFinder
+geisthaus-pagefetcher: GeistHaus-PageFetcher
+gemini-deep-research: Gemini-Deep-Research
+genomecrawlerd: GenomeCrawlerd
+getthit.com: getthit.com
+gg peekbot: GG PeekBot
+giftedvisitor scan: GIFTEDVISITOR SCAN
+gigablast: Gigablast
+gingercrawler: GingerCrawler
+github-camo: github-camo
+globalwebsearch: GlobalWebSearch
+gluten free crawler/: Gluten Free Crawler
+gnam gnam spider: gnam gnam spider
+gnowitnewsbot: GnowitNewsbot
+golfe/: Golfe
+goodreads: Goodreads
+goodzer: Goodzer
+google favicon: Google Favicon
+google trust services: Google Trust Services
+google-ads-conversions: Google-Ads-Conversions
+google-adwords-express: Google-AdWords-Express
+google-adwords-instant: Google-Adwords-Instant
+google-apps-script: Google-Apps-Script
+google-area120: Google-Area120
+google-certificates-bridge: Google-Certificates-Bridge
+google-firebase: Google-Firebase
+google-inspectiontool: Google-InspectionTool
+google-physicalweb: Google-PhysicalWeb
+google-read-aloud: Google-Read-Aloud
+google-safety: Google-Safety
+google-trust-services/: Google-Trust-Services
+googleagent-urlcontext: GoogleAgent-URLContext
+googleassociationservice: GoogleAssociationService
+googledocs: GoogleDocs
+googleproducer: GoogleProducer
+googlestackdrivermonitoring: GoogleStackdriverMonitoring
+goparserbot: GoParserBot
+gopay: GoPay
+gotosocial: gotosocial
+gotsitemonitor.com: GotSiteMonitor.com
+gowikibot: Gowikibot
+grammarly/: Grammarly
+gregcrawler: gregcrawler
+greppr web crawler: Greppr Web Crawler
+grobbot: Grobbot
+grok bot: Grok Bot
+grokappandroid: GrokAppAndroid
+grokbot: GrokBot
+groovinaadsbot/: GroovinaAdsbot
+grouphigh/: GroupHigh
+groupmebot: GroupMeBot
+grover/: Grover
+grub.org: grub.org
+gsa-crawler: gsa-crawler
+gslfbot: gslfbot
+gtmetrix: GTmetrix
+guestpostsbot/: GuestpostsBot
+gulper web bot: Gulper Web Bot
+halobot: HaloBot
+hanaleibot: HanaleiBot
+happywing: HappyWing
+hardenize: Hardenize
+harkbot: HarkBot
+harsilbot/: harsilbot
+hatena: Hatena
+hawaiibot: HawaiiBot
+hcardvalidator: hCardValidator
+headonlyscraper: HeadOnlyScraper
+hello world: Hello World
+helloworkjobpostingbot/: HelloworkJobPostingBot
+henkbot: HenkBot
+hetrixtools: HetrixTools
+hgfalphaxcrawl/: hgfAlphaXCrawl
+hifibot/: HIFIBot
+hlidam.to robot: Hlidam.to robot
+honeybadger uptime check: Honeybadger Uptime Check
+honeybadgerbot: HoneybadgerBot
+hosttracker/: HostTracker
+hotjar: Hotjar
+hstspreload-bot: hstspreload-bot
+http banner detection: HTTP Banner Detection
+http_get: http_get
+httpunit: httpunit
+httpurlconnection: HttpUrlConnection
+httpx: httpx
+hunyuan: Hunyuan
+hydrozen.io: Hydrozen.io
+hype machine/: Hype Machine
+hypestat: hypestat
+hyscore.io: hyscore.io
+ias crawler: IAS crawler
+iaskbot: iAskBot
+iaskspider: iaskspider
+ibm crawler: IBM Crawler
+iboubot: IbouBot
+icbot/: ICBot
+icc-crawler: ICC-Crawler
+ichiro: ichiro
+idg/eu: IDG/EU
+iframely: Iframely
+ifttt/: IFTTT
+imagefetcher/: ImageFetcher
+imagemind: ImageMind
+imagesiftbot: ImagesiftBot
+imagespider: imageSpider
+img2dataset: img2dataset
+imgproxy/: imgproxy
+impact.com agent: Impact.com Agent
+impendoom-bot/: impendoom-bot
+imrbot: imrbot
+indeedbot: IndeedBot
+indeedjobbot: IndeedJobBot
+indy library: Indy Library
+inetdex-bot: INETDEX-BOT
+infoobot: infoobot
+infosifter: InfoSifter
+innguma/: Innguma
+innologica: Innologica
+inoreader: inoreader
+integromat/: Integromat
+integromedb: integromedb
+intelium_bot: intelium_bot
+intelx.io_bot: intelx.io_bot
+interfaxscanbot: InterfaxScanBot
+internetmeasurement: InternetMeasurement
+internetvista monitor: internetVista monitor
+ioncrawl: IonCrawl
+ip-web-crawler.com: ip-web-crawler.com
+ips-agent: ips-agent
+irlbot: IRLbot
+irokez.cz monitoring: Irokez.cz monitoring
+isdownbot/: IsDownBot
+iskanie: iskanie
+isonai-user: IsonAI-User
+isonsearchbot: IsonSearchBot
+isscyberriskcrawler: ISSCyberRiskCrawler
+it2media-domain-crawler: it2media-domain-crawler
+iubenda-radar/: iubenda-radar
+james bot: James BOT
+jamie's spider: Jamie's Spider
+jetmon/: jetmon
+jetty: Jetty
+jobboersebot: JobboerseBot
+jobswithgptcom-bot: jobswithgptcom-bot
+jpg-newsbot: jpg-newsbot
+jugendschutzprogramm-crawler: Jugendschutzprogramm-Crawler
+jumio: Jumio
+jyxobot: jyxobot
+k7mlwcbot: K7MLWCBot
+kagi-fetcher: kagi-fetcher
+kagibot/: Kagibot
+kaikki.org-digital-archive: kaikki.org-digital-archive
+kangaroo bot: Kangaroo Bot
+kangaroobot/: KangarooBot
+kargobot-artemis: KargoBot-Artemis
+kazbtbot/: kazbtbot
+kb.dk_bot: kb.dk_bot
+keenablebot: KeenableBot
+keybasebot: KeybaseBot
+keycdn-tools/: keycdn-tools
+keydrop.io: Keydrop.io
+keys-so-bot: keys-so-bot
+kimi-agent: Kimi-Agent
+kimi-searchbot: Kimi-SearchBot
+kimi-user: Kimi-User
+kimibot: KimiBot
+kinsta-bot: kinsta-bot
+klaviyo/: Klaviyo
+klaviyoaibot: KlaviyoAIBot
+known agent: Known Agent
+konturbot/: Konturbot
+kosmiobot: KosmioBot
+krawlerbot: KrawlerBot
+kstandbot: KStandBot
+kukei.eu-bot/: Kukei.eu-Bot
+kunatocrawler: KunatoCrawler
+l9explore: l9explore
+l9scan: l9scan
+lac_iaharvester: LAC_IAHarvester
+laion-huggingface-processor: laion-huggingface-processor
+laiondownloader: LAIONDownloader
+landau-media-spider: Landau-Media-Spider
+larbin: larbin
+laserlikebot: Laserlikebot
+lastmodbot/: LastModBot
+lb-spider: lb-spider
+legalmonster: LegalMonster
+leikibot: Leikibot
+lemmy: Lemmy
+let's encrypt: Let's Encrypt
+level9searchbot/: Level9SearchBot
+library of congress web archiving: Library Of Congress Web Archiving
+lightspeedsystemscrawler: LightspeedSystemsCrawler
+linerbot: LinerBot
+linespider: Linespider
+linguee bot: Linguee Bot
+linkace/: LinkAce
+linkarchiver: LinkArchiver
+linkcheckerbot: LinkCheckerBot
+linkdex: linkdex
+linkisbot: LinkisBot
+linko: linko
+linkpadbot: LinkpadBot
+linkreader/: linkReader
+linksindexerbot/: LinksIndexerBot
+linktiger: LinkTiger
+linkupbot: LinkupBot
+linkwalker/: LinkWalker
+listgembot: ListGemBot
+loc.gov/programs/web-archiving: loc.gov/programs/web-archiving
+logicmonitor: LogicMonitor
+loomlybot: LoomlyBot
+loopimprovements.com: loopimprovements.com
+lssbot: lssbot
+lssrocketcrawler: lssrocketcrawler
+luminator-robots: Luminator-robots
+lyonl-asset-proxy: lyonl-asset-proxy
+lyonl-crawler: lyonl-crawler
+macocu: MaCoCu
+macrobondbot: Macrobondbot
+madbbot/: MADBbot
+magellan: Magellan
+magibot: MagiBot
+magicsearchdev/: magicsearchdev
+magnet.me-web/: Magnet.me-web
+magnetmebot: MagnetmeBot
+magpierss: MagpieRSS
+magus bot: Magus Bot
+mail.ru: mail.ru
+mailchimp: MailChimp
+mainwp/: MainWP
+managewp: ManageWP
+marketgoo/: MarketGoo
+marketingminer: MarketingMiner
+masscan: masscan
+matchorysearch: MatchorySearch
+mattermost-bot/: Mattermost-Bot
+mavifinds: Mavifinds
+mb-linkchecker: MB-LinkChecker
+mbcrawler: MBCrawler
+mechanize: mechanize
+mediamonitoringbot/: MediaMonitoringBot
+mediatoolkitbot: Mediatoolkitbot
+mediavinemetadataparser/: MediavineMetadataParser
+megaindex: MegaIndex
+meltwaternews: MeltwaterNews
+mergadobot: MergadoBot
+meta-externalads/: meta-externalads
+meta-externalagent: Meta-ExternalAgent
+meta-externalfetcher: Meta-ExternalFetcher
+meta-externalhit/: Meta-ExternalHit
+meta-webindexer: meta-webindexer
+metaiab facebook: MetaIAB Facebook
+metainspector: MetaInspector
+metajobbot: MetaJobBot
+metorik: Metorik
+mgidbot: MgidBot
+microsoftpreview/: MicrosoftPreview
+mindupbot: mindUpBot
+miniature.io/: Miniature.io
+minicrawler: minicrawler
+miniflux/: Miniflux
+minoru's fediverse crawler: Minoru's Fediverse Crawler
+mirrorweb.com: mirrorweb.com
+mirrorwebcrawler: MirrorWebCrawler
+missinglettrbot/: MissinglettrBot
+mistralai-index: MistralAI-Index
+mistralai-user: MistralAI-User
+mithril-crawler: mithril-crawler
+mixnodecache/: MixnodeCache
+mixrankbot: MixrankBot
+mlbot: mlbot
+moatbot: moatbot
+modatscanner: ModatScanner
+modernizebot: ModernizeBot
+modularconnector/: ModularConnector
+mollie http client: Mollie HTTP client
+monibot: Monibot
+monitis -: monitis -
+monitoring360bot/: monitoring360bot
+monitorss/: MonitoRSS
+monsidobot: Monsidobot
+monspark/: MonSpark
+montastic-monitor: montastic-monitor
+montasticmonitor: MontasticMonitor
+montools.com: MonTools.com
+moodlebot: MoodleBot
+moonshotbot: MoonshotBot
+moreover: Moreover
+motominerbot/: MotoMinerBot
+mozilla-tabstack: Mozilla-Tabstack
+mrgbot/: MRGbot
+msrbot: msrbot
+mtrobot: MTRobot
+muckrack: MuckRack
+multiviewbot: Multiviewbot
+mxtoolbox: MxToolbox
+my-tiny-bot: my-tiny-bot
+mybot/: MyBot
+mycentralaiscraperbot: MyCentralAIScraperBot
+nagetbot: NagetBot
+nanointeractive: NanoInteractive
+napbot: NapBot
+naver blog rssbot: NAVER Blog Rssbot
+naverbot: NaverBot
+nbertaupete95: nbertaupete95
+nerdbynature.bot: NerdByNature.Bot
+nessus: Nessus
+nestdaddybot: NestDaddybot
+netapi: NetAPI
+netestate imprint crawler: netEstate Imprint Crawler
+netestate ne crawler: netEstate NE Crawler
+neticle crawler: Neticle Crawler
+netpeakcheckerbot/: NetpeakCheckerBot
+netresearchserver: netresearchserver
+netseer crawler: NetSeer crawler
+netshelter contentscan: NetShelter ContentScan
+netsystemsresearch: NetSystemsResearch
+netumo: Netumo
+netvigie: NETVIGIE
+new york times newsgathering: New York Times Newsgathering
+newrelicsynthetics: NewRelicSynthetics
+newsai: newsai
+newsblur: NewsBlur
+newsharecounts: newsharecounts
+newsify feed fetcher: Newsify Feed Fetcher
+newsnow/: NewsNow
+newspaper/: newspaper
+newsroom.bi: NewsRoom.BI
+nicecrawler: Nicecrawler
+nicheindex: NicheIndex
+nikto: Nikto
+nimbostratus-bot: Nimbostratus-Bot
+ninja bot: NINJA bot
+nitro-: Nitro-
+nitrobot: NitroBot
+nixstatsbot: NIXStatsbot
+nlnz_iaharvester: NLNZ_IAHarvester
+nlux_iaharvester: NLUX_IAHarvester
+nmap scripting engine: Nmap Scripting Engine
+noahbot: NoahBot
+node-fetch: node-fetch
+nodeping: NodePing
+noibu: Noibu
+nomore404.com robot: nomore404.com robot
+noorobot: noorobot
+nooshub/: Nooshub
+nostocrawlerbot: NostoCrawlerBot
+notebooklm: NotebookLM
+notion/: Notion
+novaact: NovaAct
+novellum: Novellum
+nstoolsbot/: NsToolsBot
+ntentbot: NTENTbot
+nuzzel: Nuzzel
+nvdorz: nvdorz
+nyt_scraping: nyt_scraping
+oai-adsbot: OAI-AdsBot
+oai-searchbot: OAI-SearchBot
+observepoint: ObservePoint
+odklbot/: OdklBot
+officestorebot: officestorebot
+offline explorer: Offline Explorer
+ohdear/: OhDear
+oicrawler: OICrawler
+okhttp: okhttp
+omnisend/: Omnisend
+oncrawl/: OnCrawl
+onetrust: OneTrust
+online domain tools: Online Domain Tools
+onlineornot.com_bot: OnlineOrNot.com_bot
+opencode: opencode
+opengraph.io/: OpenGraph.io
+opengraphcheck: OpenGraphCheck
+openindexspider: openindexspider
+openrss: OpenRSS
+opentheboxbot: OpenTheBoxBot
+openvas: OpenVAS
+operator/: Operator
+optimizer): Optimizer)
+orbbot/: Orbbot
+orlo-linkpreview/: Orlo-LinkPreview
+outbrain: outbrain
+outclicksbot: OutclicksBot
+outsellurlvalidator: OutsellURLValidator
+overcast/: Overcast
+owler (ows.eu: Owler (ows.eu
+owler-w: OWLer-W
+page-preview-tool: page-preview-tool
+page2rss: page2rss
+pagepeeker/: PagePeeker
+pagespeed/: Pagespeed
+pagething: PageThing
+pandalytics: Pandalytics
+pangubot: PanguBot
+panscient: Panscient
+paqlebot/: Paqlebot
+parse.ly scraper/: parse.ly scraper
+parselysharesbot: ParselySharesBot
+payhawk invoice fetching agent: Payhawk Invoice Fetching Agent
+paypal/: PayPal
+pcore-http: Pcore-HTTP
+pdf24 url to pdf: PDF24 URL To PDF
+peer39_crawler: peer39_crawler
+penthouse critical path css generator: Penthouse Critical Path CSS Generator
+perplexity-user: Perplexity-User
+perplexityuser: PerplexityUser
+petalbot: PetalBot
+phindbot: PhindBot
+phpcrawl: phpcrawl
+phxbot: PhxBot
+pingadmin.ru/: PingAdmin.Ru
+pingping.io/: pingping.io
+piplbot: PiplBot
+pixalate.com: Pixalate.com
+plagawarebot: PlagAwareBot
+playstore-google: PlayStore-Google
+plesk screenshot bot: Plesk screenshot bot
+pocketcasts/: PocketCasts
+pocketparser: PocketParser
+podchaserparser: PodchaserParser
+podimo: Podimo
+poduptime: Poduptime
+poggio-citations: Poggio-Citations
+pomothy-bot: Pomothy-Bot
+poseidon research crawler: Poseidon Research Crawler
+postrank: postrank
+potions/: Potions
+pressenginebot: PressEngineBot
+pricedroneshoppingbot/: PricedroneShoppingBot
+priecobot/: PriEcoBot
+primalbot: Primalbot
+printfriendly.com: PrintFriendly.com
+pro-sitemaps/: Pro-Sitemaps
+probelyspdr/: ProbelySPDR
+productsup.io/crawler: productsup.io/crawler
+projectshield-urlcheck: ProjectShield-UrlCheck
+protopage/: Protopage
+prtg network monitor: PRTG Network Monitor
+prtgcloudbot/: PRTGCloudBot
+ps_daily/: PS_Daily
+psbot: psbot
+pulsepoint: Pulsepoint
+pulsetic.com: pulsetic.com
+purebot: purebot
+pwabuilderhttpagent: PWABuilderHttpAgent
+python-opengraph: python-opengraph
+pywikibot/: Pywikibot
+qcbot: qcbot
+qodercli: qodercli
+qualifiedbot: QualifiedBot
+qualys: Qualys
+quantcastbot/: Quantcastbot
+querit-searchbot: Querit-SearchBot
+queritbot: QueritBot
+quillbot: QuillBot
+quora-bot: Quora-Bot
+qwantbot: Qwantbot
+qwantify: Qwantify
+qwarrybot: Qwarrybot
+qwenbot: QwenBot
+rackspace monitoring/: Rackspace Monitoring
+rakuten image extraction bot: Rakuten Image extraction bot
+rankactivelinkbot: RankActiveLinkBot
+rankurbot/: RankurBot
+ravencrawler/: RavenCrawler
+rawweb-bot: rawweb-bot
+readable/: Readable
+reelevant/: Reelevant
+refindbot: Refindbot
+reflectionbot: Reflectionbot
+regionstuttgartbot: RegionStuttgartBot
+remove.bg/: remove.bg
+repolookoutbot: RepoLookoutBot
+researchbot: ResearchBot
+researchscan.comsys.rwth-aachen.de: researchscan.comsys.rwth-aachen.de
+retool/: Retool
+retrevopageanalyzer: RetrevoPageAnalyzer
+retrolistecom/: RetroListeCOM
+reverseengineeringbot: ReverseEngineeringBot
+revvimgort/: RevvimGort
+reward-gateway: reward-gateway
+ridderbot: RidderBot
+rigor): Rigor)
+rivva: Rivva
+robotschecker/: RobotsChecker
+rsiteauditor: RSiteAuditor
+rss reader: RSS Reader
+rss-is-dead.lol web bot: rss-is-dead.lol web bot
+rss-parser: rss-parser
+rss.social: RSS.Social
+rss2tg: rss2tg
+rssapi/: RSSAPI
+rssbot/: rssbot
+rssingbot: RSSingBot
+rssreaderbot: RssReaderBot
+ruxitsynthetic: RuxitSynthetic
+rytebot: RyteBot
+s4a-probe-bot/: s4a-probe-bot
+safednsbot: SafeDNSBot
+safesearch microdata crawler: SafeSearch microdata crawler
+salesforce.com: Salesforce.com
+sandobacrawler/: SandobaCrawler
+sansec security monitor/: Sansec Security Monitor
+sbintuitionsbot: SBIntuitionsBot
+sbl-bot: SBL-BOT
+schema-markup-validator: Schema-Markup-Validator
+scoop.it/: Scoop.it
+scope3: Scope3
+scoutjet: ScoutJet
+scrapeherobot/: ScrapeheroBot
+scraping@nytimes.com: scraping@nytimes.com
+screeenly-bot: screeenly-bot
+scribdbot: scribdbot
+scrubby: Scrubby
+scrybot: ScryBot
+search.marginalia.nu: search.marginalia.nu
+searchatlas: SearchAtlas
+searcherweb: Searcherweb
+searcherxweb: Searcherxweb
+searchexpress: SearchExpress
+sebot-wa: SEBot-WA
+securityheaders: SecurityHeaders
+seekbot: Seekbot
+seekport: Seekport
+seewithkids: seewithkids
+semaltbot/: semaltbot
+semantic-visions: semantic-visions
+semanticscholarbot: SemanticScholarBot
+sempi.tech: sempi.tech
+sendgrid event api: SendGrid Event API
+sentibot: SentiBot
+sentinel-linkcheck: SENTINEL-LinkCheck
+sentry/: sentry
+senutobot: SenutoBot
+seo4ajax.com: seo4ajax.com
+seolizer: SEOlizer
+seolyt: seoLyt
+seoscanners: seoscanners
+seostar.co: seostar.co
+seoulbot: SeoulBot
+sequelwp: SequelWP
+serankingbacklinksbot: SERankingBacklinksBot
+serendeputybot: SerendeputyBot
+serpstatbot/: serpstatbot
+serptimizerbot: SERPtimizerBot
+server density service monitoring: Server Density Service Monitoring
+serverdensity: ServerDensity
+serverhunterspider/: ServerHunterSpider
+seznamhomepagecrawler/: SeznamHomepageCrawler
+sfdc-callout/: SFDC-Callout
+shap-user: Shap-User
+shapbot: ShapBot
+shopify-captain-hook: Shopify-Captain-Hook
+shortpixel: ShortPixel
+shortwave image fetcher: Shortwave Image Fetcher
+sidetrade indexer bot: Sidetrade indexer bot
+silktide: Silktide
+simplecrawler: SimpleCrawler
+simplepie: SimplePie
+simplescraper: SimpleScraper
+sincerasyntheticuser/: SinceraSyntheticUser
+sindresorhus/got: sindresorhus/got
+sindup: Sindup
+site24x7: Site24x7
+siteauditbot/: SiteAuditBot
+sitebot: sitebot
+sitebulb: sitebulb
+siteexplorer.info: siteexplorer.info
+siteimprove.com: Siteimprove.com
+sitelock: SiteLock
+sitescorebot: SiteScoreBot
+sitesearch360/: SiteSearch360
+sitesoverpagesbot: SitesOverPagesBot
+sitesucker: SiteSucker
+skroutz imagebot: Skroutz ImageBot
+skroutzbot: SkroutzBot
+skyworkspider: SkyworkSpider
+slaccalebot: SlaccaleBot
+sleepbot: SleepBot
+slickbot/: SlickBot
+smarshbot: SmarshBot
+smartologybot/: SmartologyBot
+smtnetpmbot: SMTnetPMBot
+snacktory: Snacktory
+snap url preview service: Snap URL Preview Service
+snapurlpreview/: SnapURLPreview
+snipcart/: Snipcart
+software-security-research: Software-Security-Research
+solarwinds/: solarwinds
+sora pos/: Sora POS
+sosospider: Sosospider
+sottopopnone: SottopopNone
+sparkpost: SparkPost
+sparkshipping: SparkShipping
+specificfeeds: Specificfeeds
+spectate/: Spectate
+spider.com: spider.com
+spiderling: SpiderLing
+splunk: Splunk
+springservebot: SpringserveBot
+sqlmap: sqlmap
+sqwatcher: SQWatcher
+ssl labs: SSL Labs
+ssssbot/: SSSSBot
+stape/: Stape
+startmebot/: startmebot
+startpageprivateimageproxy/: StartpagePrivateImageProxy
+statistikaustria/: StatistikAustria
+statsdronebot: StatsDroneBot
+statusnestbacklinkspider: StatusNestBacklinkSpider
+stepstonecrawlbot: stepstoneCrawlBot
+storebot-google: Storebot-Google
+storygizebot: StorygizeBot
+stractbot: StractBot
+streamline3bot/: Streamline3Bot
+stripe/: Stripe
+sucuri: Sucuri
+summalybot: SummalyBot
+summify: summify
+supabase paired crawler: Supabase Paired Crawler
+superfeedr: Superfeedr
+surdotlybot: SurdotlyBot
+surveybot: SurveyBot
+sustainabilitycrawler: SustainabilityCrawler
+sv-watchagent: sv-watchagent
+svix-webhooks/: Svix-Webhooks
+swifteqlinkchecker: SwifteqLinkChecker
+swimgbot: SWIMGBot
+swisscows: Swisscows
+synapse: Synapse
+synthesibot: SynthesiBot
+synthetic-monitoring-agent/: synthetic-monitoring-agent
+sysomos: Sysomos
+t3versionsbot: t3versionsBot
+tactiscout/: TactiScout
+tagoobot: tagoobot
+taragroup intelligent bot: TaraGroup Intelligent Bot
+tavilybot: TavilyBot
+tchelebi/: tchelebi
+telegrambot: TelegramBot
+teoma: Teoma
+termly: Termly
+terra cotta: Terra Cotta
+terracotta: TerraCotta
+test-bot: test-bot
+testcrawler: Testcrawler
+testuri: TestURI
+textrazor: TextRazor
+theinternetsearchx: TheInternetSearchx
+theoldreader.com: theoldreader.com
+thesis-research-bot: thesis-research-bot
+thinkbot: Thinkbot
+thinklab: Thinklab
+thousandeyes: ThousandEyes
+tigerbot: tigerbot
+tiktokspider: TikTokSpider
+timpibot: Timpibot
+tiny tiny rss: Tiny Tiny RSS
+tls tester: TLS tester
+tombapublicwebcrawler: TombaPublicWebCrawler
+top100.rambler.ru crawler: top100.rambler.ru crawler
+toplistbot: toplistbot
+toutiaospider: ToutiaoSpider
+tracemyfile: tracemyfile
+tracking-quality-spider: tracking-quality-spider
+trafilatura: trafilatura
+trellis-services: Trellis-Services
+trentwil.es: trentwil.es
+trove: Trove
+trovitbot: trovitBot
+trustly/: Trustly
+tsm-turingos: TSM-turingos
+tsmbot: TSMbot
+ttd-content: TTD-Content
+tumblr/: Tumblr
+turnitin: Turnitin
+tweakers: Tweakers
+twilioproxy/: TwilioProxy
+twingly: Twingly
+twinwavescanner: TwinWaveScanner
+uaslinkchecker/: UASlinkChecker
+ugaresearchagent: UGAResearchAgent
+uipbot: uipbot
+unshorten.it!: Unshorten.It!
+upday/: upday
+upflow: Upflow
+uptime-kuma: Uptime-Kuma
+uptime/: Uptime
+uptimebot.org: Uptimebot.org
+uptimebot/: UptimeBot
+uptimedoctor: uptimedoctor
+uptimestatistics: UptimeStatistics
+uptimia: Uptimia
+uptrends: uptrends
+urlappendbot: urlappendbot
+urlbeebot: UrlBeeBot
+urlcheckr/: Urlcheckr
+urlsuma.de crawler: UrlSuMa.de crawler
+useebookchecker/: useeBookChecker
+usinenouvellecrawler: UsineNouvelleCrawler
+ut-dorkbot: UT-Dorkbot
+validator.nu: Validator.nu
+valve/steam: Valve/Steam
+vaultpress: VaultPress
+vebidoobot: vebidoobot
+velenpublicwebcrawler: VelenPublicWebCrawler
+veoozbot: Veoozbot
+vercel-screenshot: vercel-screenshot
+vercelbot: Vercelbot
+verispider: Verispider
+verity/: Verity
+videootv bot: videootv Bot
+videootvbot: videootvBot
+virustotal: virustotal
+visionheight.com/scan: visionheight.com/scan
+vkrobot: VKRobot
+vmcrawl: vmcrawl
+voluumdsp-content-bot: VoluumDSP-content-bot
+vsusearchspider/: VsuSearchSpider
+vu-server-health-scanner/: vu-server-health-scanner
+vuhuvbot: vuhuvBot
+w3c-checklink: W3C-checklink
+w3c-mobileok: W3C-mobileOK
+w3c_css_validator: W3C_CSS_Validator
+w3c_i18n-checker: W3C_I18n-Checker
+w3c_unicorn: W3C_Unicorn
+w3c_validator: W3C_Validator
+wadoobot: WadooBot
+wanscannerbot: WanscannerBot
+wardbot: WARDBot
+watchbot monitoring robot: Watchbot monitoring robot
+watchforbot: WatchForBot
+watchful: Watchful
+watchmouse: WatchMouse
+watchtowr: watchTowr
+wayback: Wayback
+web measure/: Web Measure
+web screen service by hyperhost: Web Screen Service By hyperhost
+web-archive-net.com.bot: web-archive-net.com.bot
+webcapture: WebCapture
+webcompanycrawler: webcompanycrawler
+webcopier: WebCopier
+webdatastats: WebDataStats
+webflow: Webflow
+webgains-bot: webgains-bot
+webmon: webmon
+weborama-fetcher: weborama-fetcher
+webprosbot/: webprosbot
+website-info.net-robot: Website-info.net-Robot
+websiteops: WebsiteOps
+websitepulse: websitepulse
+websniffer/: WebSniffer
+webspidermount: webspidermount
+webwikibot/: WebwikiBot
+webzio: webzio
+webzip: WebZIP
+wedos online: WEDOS OnLine
+wellknownbot: WellKnownBot
+wepchsearchengine: WepchSearchEngine
+whatsmyip.org: WhatsMyIP.org
+whatweb/: WhatWeb
+wheregoes.com: Wheregoes.com
+wheresitup.com/: wheresitup.com
+wirereaderbot/: WireReaderBot
+wisebot: WISEbot
+wjhro/: WJHRO
+wknd-bot: wknd-bot
+wmtips.com/: wmtips.com
+wocbot: wocbot
+woobot: woobot
+wordcountbot/: WordCountBot
+wordup-1: Wordup-1
+wordupinfosearch: WordupInfoSearch
+workona-favicon-service/: workona-favicon-service
+worldbot: WorldBot
+wovn crawler: WOVN Crawler
+wovncrawler/: WovnCrawler
+wowlink crawler/: wowLink Crawler
+wp time capsule: WP Time Capsule
+wpbot: wpbot
+wpmu dev broken link checker: WPMU DEV Broken Link Checker
+wpmu dev hub: WPMU DEV Hub
+wpmudev uptime monitor: WPMUDEV Uptime Monitor
+wpscan: WPScan
+wpsec/: WPSec
+wpumbrella: WPUmbrella
+wrtnbot: WRTNBot
+wtotem: WTotem
+www.uptime.com: www.uptime.com
+xai-grok: xAI-Grok
+xing bot: Xing Bot
+xml sitemaps generator: XML Sitemaps Generator
+xovionpagecrawler: XoviOnpageCrawler
+xy-archive-compliance: XY-Archive-Compliance
+yahoo ad monitoring: Yahoo Ad monitoring
+yahoo link preview: Yahoo Link Preview
+yahoocachesystem: YahooCacheSystem
+yahoomailproxy: YahooMailProxy
+yandorirssbot: YandoriRSSBot
+yellowbrandprotectionbot: Yellowbrandprotectionbot
+yelpspider: yelpspider
+yextbot/: YextBot
+yibot: YiBot
+yisouspider: YisouSpider
+yiyanbot: YiyanBot
+ylt chrome: YLT Chrome
+yokoygroupag/: YokoyGroupAG
+yoozbot: yoozBot
+youdaobot: YoudaoBot
+yuuperbot: Yuuperbot
+zapier: Zapier
+zealbot: Zealbot
+zebra-v2-bot: zebra-v2-bot
+zenback bot: zenback bot
+zendesk webhook: Zendesk Webhook
+zgrab: zgrab
+zombiebot/: Zombiebot
+zoominfo-: ZoomInfo-
+zoterotranslationserver/wmf: ZoteroTranslationServer/WMF
+zuperlistbot/: ZuperlistBot
+zzhbot: zzhbot