@pug-sh/crawlers 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +205 -2
- package/dist/agents.generated.d.ts +3 -0
- package/dist/agents.generated.js +791 -0
- package/dist/crawlers.d.ts +55 -0
- package/dist/crawlers.js +222 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +1 -0
- package/dist/match.d.ts +13 -0
- package/dist/match.js +59 -0
- package/package.json +60 -4
|
@@ -0,0 +1,791 @@
|
|
|
1
|
+
// AUTOGENERATED: do not edit. Run `make agents` to regenerate.
|
|
2
|
+
// table: pug's crawler table (proto/common/crawlers/v1), as [name, category, ...lowercased tokens].
|
|
3
|
+
// listed: crawler-user-agents@1.66.0, its search-engine and seo entries, as [name, category, pattern],
|
|
4
|
+
// under its license:
|
|
5
|
+
//
|
|
6
|
+
// The MIT License (MIT)
|
|
7
|
+
//
|
|
8
|
+
// Copyright (c) 2017 Martin Monperrus
|
|
9
|
+
//
|
|
10
|
+
// Permission is hereby granted, free of charge, to any person obtaining a copy of
|
|
11
|
+
// this software and associated documentation files (the "Software"), to deal in
|
|
12
|
+
// the Software without restriction, including without limitation the rights to
|
|
13
|
+
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
|
14
|
+
// the Software, and to permit persons to whom the Software is furnished to do so,
|
|
15
|
+
// subject to the following conditions:
|
|
16
|
+
//
|
|
17
|
+
// The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
// copies or substantial portions of the Software.
|
|
19
|
+
//
|
|
20
|
+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
|
22
|
+
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
|
23
|
+
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
|
24
|
+
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
25
|
+
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
26
|
+
export const table = [
|
|
27
|
+
["ChatGPT-User", "answers", "chatgpt-user"],
|
|
28
|
+
["Claude-User", "answers", "claude-user"],
|
|
29
|
+
["Perplexity-User", "answers", "perplexity-user"],
|
|
30
|
+
["MistralAI-User", "answers", "mistralai-user"],
|
|
31
|
+
["Kimi-User", "answers", "kimi-user"],
|
|
32
|
+
["DuckAssistBot", "answers", "duckassistbot"],
|
|
33
|
+
["Amzn-User", "answers", "amzn-user"],
|
|
34
|
+
["meta-externalfetcher", "answers", "meta-externalfetcher"],
|
|
35
|
+
["Google-Agent", "answers", "google-agent"],
|
|
36
|
+
["Google-GeminiNotebook", "answers", "google-gemininotebook", "google-notebooklm"],
|
|
37
|
+
["Google-Read-Aloud", "answers", "google-read-aloud"],
|
|
38
|
+
["OAI-SearchBot", "search", "oai-searchbot"],
|
|
39
|
+
["Claude-SearchBot", "search", "claude-searchbot"],
|
|
40
|
+
["PerplexityBot", "search", "perplexitybot"],
|
|
41
|
+
["MistralAI-Index", "search", "mistralai-index"],
|
|
42
|
+
["Kimi-SearchBot", "search", "kimi-searchbot"],
|
|
43
|
+
["Amzn-SearchBot", "search", "amzn-searchbot"],
|
|
44
|
+
["meta-webindexer", "search", "meta-webindexer"],
|
|
45
|
+
["Googlebot", "search", "googlebot"],
|
|
46
|
+
["bingbot", "search", "bingbot", "msnbot"],
|
|
47
|
+
["Applebot", "search", "applebot"],
|
|
48
|
+
["Baiduspider", "search", "baiduspider"],
|
|
49
|
+
["YouBot", "search", "youbot"],
|
|
50
|
+
["GPTBot", "training", "gptbot"],
|
|
51
|
+
["ClaudeBot", "training", "claudebot"],
|
|
52
|
+
["meta-externalagent", "training", "meta-externalagent"],
|
|
53
|
+
["Amazonbot", "training", "amazonbot"],
|
|
54
|
+
["KimiBot", "training", "kimibot"],
|
|
55
|
+
["CCBot", "training", "ccbot"],
|
|
56
|
+
["Bytespider", "training", "bytespider"],
|
|
57
|
+
["AI2Bot", "training", "ai2bot"],
|
|
58
|
+
["OAI-AdsBot", "other", "oai-adsbot"],
|
|
59
|
+
["meta-externalads", "other", "meta-externalads"],
|
|
60
|
+
["GoogleOther", "other", "googleother"],
|
|
61
|
+
["Google-CloudVertexBot", "other", "google-cloudvertexbot"],
|
|
62
|
+
];
|
|
63
|
+
export const listed = [
|
|
64
|
+
["APIs-Google", "search", "APIs-Google"],
|
|
65
|
+
["Google-InspectionTool", "search", "Google-InspectionTool"],
|
|
66
|
+
["Storebot-Google", "search", "Storebot-Google"],
|
|
67
|
+
["Slurp", "search", "Slurp"],
|
|
68
|
+
["Nutch", "search", "Nutch"],
|
|
69
|
+
["jyxobot", "search", "jyxobot"],
|
|
70
|
+
["FAST-WebCrawler", "search", "FAST-WebCrawler"],
|
|
71
|
+
["FAST Enterprise Crawler", "search", "FAST Enterprise Crawler"],
|
|
72
|
+
["BIGLOTRON", "search", "BIGLOTRON"],
|
|
73
|
+
["Teoma", "search", "Teoma"],
|
|
74
|
+
["convera", "search", "convera"],
|
|
75
|
+
["Seekbot", "search", "^Seekbot"],
|
|
76
|
+
["Gigabot", "search", "Gigabot"],
|
|
77
|
+
["Gigablast", "search", "Gigablast"],
|
|
78
|
+
["exabot", "search", "exabot"],
|
|
79
|
+
["GingerCrawler", "seo", "GingerCrawler"],
|
|
80
|
+
["grub.org", "search", "grub\\.org"],
|
|
81
|
+
["UsineNouvelleCrawler", "search", "UsineNouvelleCrawler"],
|
|
82
|
+
["netresearchserver", "seo", "netresearchserver"],
|
|
83
|
+
["speedy", "search", "speedy"],
|
|
84
|
+
["fluffy", "search", "fluffy"],
|
|
85
|
+
["findlink", "seo", "findlink"],
|
|
86
|
+
["msrbot", "search", "msrbot"],
|
|
87
|
+
["panscient", "seo", "panscient"],
|
|
88
|
+
["yacybot", "search", "yacybot"],
|
|
89
|
+
["AISearchBot", "search", "AISearchBot"],
|
|
90
|
+
["ips-agent", "seo", "ips-agent"],
|
|
91
|
+
["tagoobot", "search", "tagoobot"],
|
|
92
|
+
["MJ12bot", "seo", "MJ12bot"],
|
|
93
|
+
["woriobot", "search", "woriobot"],
|
|
94
|
+
["yanga", "search", "yanga"],
|
|
95
|
+
["buzzbot", "seo", "buzzbot"],
|
|
96
|
+
["mlbot", "seo", "mlbot"],
|
|
97
|
+
["yandex.com/bots", "search", "yandex\\.com\\/bots"],
|
|
98
|
+
["purebot", "search", "purebot"],
|
|
99
|
+
["Linguee Bot", "search", "Linguee Bot"],
|
|
100
|
+
["voilabot", "search", "voilabot"],
|
|
101
|
+
["spbot", "seo", "spbot"],
|
|
102
|
+
["twengabot", "seo", "twengabot"],
|
|
103
|
+
["postrank", "seo", "postrank"],
|
|
104
|
+
["scribdbot", "search", "scribdbot"],
|
|
105
|
+
["sitebot", "seo", "sitebot"],
|
|
106
|
+
["linkdex", "seo", "linkdex"],
|
|
107
|
+
["Adidxbot", "search", "Adidxbot"],
|
|
108
|
+
["ezooms", "seo", "ezooms"],
|
|
109
|
+
["dotbot", "seo", "dotbot"],
|
|
110
|
+
["Mail.RU_Bot", "search", "Mail\\.RU_Bot"],
|
|
111
|
+
["discobot", "search", "discobot"],
|
|
112
|
+
["findthatfile", "search", "findthatfile"],
|
|
113
|
+
["NerdByNature.Bot", "seo", "NerdByNature\\.Bot"],
|
|
114
|
+
["SISTRIX Crawler", "seo", "(sistrix|SISTRIX) [cC]rawler"],
|
|
115
|
+
["AhrefsBot", "seo", "Ahrefs(Bot|SiteAudit)"],
|
|
116
|
+
["fuelbot", "seo", "fuelbot"],
|
|
117
|
+
["CrunchBot", "seo", "^CrunchBot"],
|
|
118
|
+
["IndeedBot", "search", "IndeedBot"],
|
|
119
|
+
["mappydata", "seo", "mappydata"],
|
|
120
|
+
["woobot", "seo", "woobot"],
|
|
121
|
+
["ZoominfoBot", "seo", "ZoominfoBot"],
|
|
122
|
+
["PrivacyAwareBot", "seo", "PrivacyAwareBot"],
|
|
123
|
+
["Multiviewbot", "seo", "Multiviewbot"],
|
|
124
|
+
["SWIMGBot", "seo", "SWIMGBot"],
|
|
125
|
+
["Grobbot", "seo", "Grobbot"],
|
|
126
|
+
["eright", "seo", "eright"],
|
|
127
|
+
["Apercite", "seo", "Apercite"],
|
|
128
|
+
["semanticbot", "seo", "semanticbot"],
|
|
129
|
+
["Aboundex", "search", "Aboundex"],
|
|
130
|
+
["domaincrawler", "seo", "domaincrawler"],
|
|
131
|
+
["wbsearchbot", "search", "wbsearchbot"],
|
|
132
|
+
["summify", "seo", "summify"],
|
|
133
|
+
["edisterbot", "search", "edisterbot"],
|
|
134
|
+
["SeznamBot", "search", "SeznamBot"],
|
|
135
|
+
["gslfbot", "seo", "gslfbot"],
|
|
136
|
+
["aiHitBot", "search", "aiHitBot"],
|
|
137
|
+
["intelium_bot", "seo", "intelium_bot"],
|
|
138
|
+
["Yeti", "search", "Yeti"],
|
|
139
|
+
["RetrevoPageAnalyzer", "seo", "RetrevoPageAnalyzer"],
|
|
140
|
+
["lb-spider", "search", "lb-spider"],
|
|
141
|
+
["Sogou", "search", "Sogou"],
|
|
142
|
+
["lssbot", "seo", "lssbot"],
|
|
143
|
+
["careerbot", "seo", "careerbot"],
|
|
144
|
+
["wotbox", "search", "wotbox"],
|
|
145
|
+
["wocbot", "search", "wocbot"],
|
|
146
|
+
["ichiro", "search", "ichiro"],
|
|
147
|
+
["DuckDuckBot", "search", "DuckDuckBot"],
|
|
148
|
+
["lssrocketcrawler", "seo", "lssrocketcrawler"],
|
|
149
|
+
["drupact", "seo", "drupact"],
|
|
150
|
+
["webcompanycrawler", "seo", "webcompanycrawler"],
|
|
151
|
+
["acoonbot", "search", "acoonbot"],
|
|
152
|
+
["openindexspider", "search", "openindexspider"],
|
|
153
|
+
["gnam gnam spider", "seo", "gnam gnam spider"],
|
|
154
|
+
["backlinkcrawler", "seo", "backlinkcrawler"],
|
|
155
|
+
["coccoc", "search", "coccoc"],
|
|
156
|
+
["content crawler spider", "seo", "content crawler spider"],
|
|
157
|
+
["toplistbot", "seo", "toplistbot"],
|
|
158
|
+
["it2media-domain-crawler", "seo", "it2media-domain-crawler"],
|
|
159
|
+
["siteexplorer.info", "seo", "siteexplorer\\.info"],
|
|
160
|
+
["elisabot", "seo", "elisabot"],
|
|
161
|
+
["arabot", "search", "arabot"],
|
|
162
|
+
["WeSEE:Search", "search", "WeSEE:Search"],
|
|
163
|
+
["niki-bot", "seo", "niki-bot"],
|
|
164
|
+
["CrystalSemanticsBot", "seo", "CrystalSemanticsBot"],
|
|
165
|
+
["rogerbot", "seo", "rogerbot"],
|
|
166
|
+
["360Spider", "search", "360Spider"],
|
|
167
|
+
["psbot", "search", "psbot"],
|
|
168
|
+
["InterfaxScanBot", "seo", "InterfaxScanBot"],
|
|
169
|
+
["g00g1e.net", "seo", "g00g1e\\.net"],
|
|
170
|
+
["urlappendbot", "seo", "urlappendbot"],
|
|
171
|
+
["brainobot", "seo", "brainobot"],
|
|
172
|
+
["fr-crawler", "search", "fr-crawler"],
|
|
173
|
+
["smtbot", "seo", "smtbot"],
|
|
174
|
+
["A6-Indexer", "search", "A6-Indexer"],
|
|
175
|
+
["OrangeBot", "search", "OrangeBot\\/"],
|
|
176
|
+
["memorybot", "seo", "memorybot"],
|
|
177
|
+
["MegaIndex", "seo", "MegaIndex"],
|
|
178
|
+
["nerdybot", "seo", "nerdybot"],
|
|
179
|
+
["xovibot", "seo", "xovibot"],
|
|
180
|
+
["Qwantify", "search", "Qwantify"],
|
|
181
|
+
["findxbot", "search", "findxbot"],
|
|
182
|
+
["SemrushBot", "seo", "S[eE][mM]rushBot"],
|
|
183
|
+
["yoozBot", "seo", "yoozBot"],
|
|
184
|
+
["lipperhey", "seo", "lipperhey"],
|
|
185
|
+
["Y!J", "search", "Y!J"],
|
|
186
|
+
["Domain Re-Animator Bot", "seo", "Domain Re-Animator Bot"],
|
|
187
|
+
["Screaming Frog SEO Spider", "seo", "Screaming Frog SEO Spider"],
|
|
188
|
+
["MetaURI", "seo", "MetaURI"],
|
|
189
|
+
["LivelapBot", "seo", "Livelap[bB]ot"],
|
|
190
|
+
["OpenHoseBot", "search", "OpenHoseBot"],
|
|
191
|
+
["collection@infegy.com", "seo", "collection@infegy\\.com"],
|
|
192
|
+
["IstellaBot", "search", "IstellaBot"],
|
|
193
|
+
["DeuSu", "search", "DeuSu\\/"],
|
|
194
|
+
["betaBot", "seo", "betaBot"],
|
|
195
|
+
["Cliqzbot", "search", "Cliqzbot\\/"],
|
|
196
|
+
["MojeekBot", "search", "MojeekBot\\/"],
|
|
197
|
+
["netEstate NE Crawler", "seo", "netEstate NE Crawler"],
|
|
198
|
+
["SafeSearch microdata crawler", "search", "SafeSearch microdata crawler"],
|
|
199
|
+
["Gluten Free Crawler", "search", "Gluten Free Crawler\\/"],
|
|
200
|
+
["Sonic", "seo", "Sonic"],
|
|
201
|
+
["Sysomos", "seo", "Sysomos"],
|
|
202
|
+
["deadlinkchecker", "seo", "deadlinkchecker"],
|
|
203
|
+
["RankActiveLinkBot", "seo", "RankActiveLinkBot"],
|
|
204
|
+
["iskanie", "seo", "iskanie"],
|
|
205
|
+
["Veoozbot", "seo", "Veoozbot"],
|
|
206
|
+
["datagnionbot", "seo", "datagnionbot"],
|
|
207
|
+
["contxbot", "seo", "contxbot"],
|
|
208
|
+
["electricmonk", "seo", "electricmonk"],
|
|
209
|
+
["GarlikCrawler", "seo", "GarlikCrawler"],
|
|
210
|
+
["vebidoobot", "seo", "vebidoobot"],
|
|
211
|
+
["FemtosearchBot", "search", "FemtosearchBot"],
|
|
212
|
+
["MetaJobBot", "seo", "MetaJobBot"],
|
|
213
|
+
["DomainStatsBot", "seo", "DomainStatsBot"],
|
|
214
|
+
["mindUpBot", "seo", "mindUpBot"],
|
|
215
|
+
["Daum", "search", "Daum\\/"],
|
|
216
|
+
["Xenu Link Sleuth", "seo", "Xenu Link Sleuth"],
|
|
217
|
+
["KosmioBot", "seo", "KosmioBot"],
|
|
218
|
+
["Gowikibot", "seo", "Gowikibot"],
|
|
219
|
+
["PiplBot", "seo", "PiplBot"],
|
|
220
|
+
["James BOT", "seo", "James BOT"],
|
|
221
|
+
["Barkrowler", "seo", "Bark[rR]owler"],
|
|
222
|
+
["TinEye", "seo", "TinEye"],
|
|
223
|
+
["SocialRankIOBot", "seo", "SocialRankIOBot"],
|
|
224
|
+
["trendictionbot", "seo", "trendictionbot"],
|
|
225
|
+
["Ocarinabot", "seo", "Ocarinabot"],
|
|
226
|
+
["epicbot", "seo", "epicbot"],
|
|
227
|
+
["Primalbot", "seo", "Primalbot"],
|
|
228
|
+
["DuckDuckGo-Favicons-Bot", "search", "DuckDuckGo-Favicons-Bot"],
|
|
229
|
+
["GnowitNewsbot", "seo", "GnowitNewsbot"],
|
|
230
|
+
["Leikibot", "seo", "Leikibot"],
|
|
231
|
+
["YaK", "seo", "YaK\\/"],
|
|
232
|
+
["PaperLiBot", "seo", "PaperLiBot"],
|
|
233
|
+
["Digg Deeper", "seo", "Digg Deeper"],
|
|
234
|
+
["dcrawl", "seo", "^dcrawl"],
|
|
235
|
+
["Snacktory", "seo", "Snacktory"],
|
|
236
|
+
["AndersPinkBot", "seo", "AndersPinkBot"],
|
|
237
|
+
["Fyrebot", "seo", "Fyrebot"],
|
|
238
|
+
["Luminator-robots", "seo", "Luminator-robots"],
|
|
239
|
+
["ExtLinksBot", "seo", "ExtLinksBot"],
|
|
240
|
+
["SurveyBot", "seo", "SurveyBot"],
|
|
241
|
+
["Nuzzel", "seo", "Nuzzel"],
|
|
242
|
+
["omgili", "seo", "omgili"],
|
|
243
|
+
["PocketParser", "seo", "PocketParser"],
|
|
244
|
+
["YisouSpider", "search", "YisouSpider"],
|
|
245
|
+
["um-LN", "seo", "um-LN"],
|
|
246
|
+
["ToutiaoSpider", "search", "ToutiaoSpider"],
|
|
247
|
+
["MuckRack", "seo", "MuckRack"],
|
|
248
|
+
["Jamie's Spider", "seo", "Jamie's Spider"],
|
|
249
|
+
["Laserlikebot", "seo", "Laserlikebot"],
|
|
250
|
+
["AppEngine-Google", "search", "AppEngine-Google"],
|
|
251
|
+
["Upflow", "seo", "Upflow"],
|
|
252
|
+
["Traackr.com", "seo", "Traackr\\.com"],
|
|
253
|
+
["Twurly", "seo", "Twurly"],
|
|
254
|
+
["DnyzBot", "seo", "DnyzBot"],
|
|
255
|
+
["botify", "seo", "botify"],
|
|
256
|
+
["007ac9 Crawler", "seo", "007ac9 Crawler"],
|
|
257
|
+
["BehloolBot", "seo", "BehloolBot"],
|
|
258
|
+
["BrandVerity", "seo", "BrandVerity"],
|
|
259
|
+
["BDCbot", "seo", "BDCbot"],
|
|
260
|
+
["ZumBot", "search", "ZumBot"],
|
|
261
|
+
["EZID", "seo", "EZID"],
|
|
262
|
+
["BLP_bbot", "seo", "BLP_bbot"],
|
|
263
|
+
["BomboraBot", "seo", "BomboraBot"],
|
|
264
|
+
["Companybook-Crawler", "seo", "Companybook-Crawler"],
|
|
265
|
+
["Genieo", "seo", "Genieo"],
|
|
266
|
+
["magpie-crawler", "seo", "magpie-crawler"],
|
|
267
|
+
["MeltwaterNews", "seo", "MeltwaterNews"],
|
|
268
|
+
["Moreover", "seo", "Moreover"],
|
|
269
|
+
["ScoutJet", "seo", "ScoutJet"],
|
|
270
|
+
["StorygizeBot", "seo", "StorygizeBot"],
|
|
271
|
+
["seoscanners", "seo", "seoscanners"],
|
|
272
|
+
["MauiBot", "seo", "MauiBot"],
|
|
273
|
+
["AlphaBot", "seo", "AlphaBot"],
|
|
274
|
+
["acapbot", "seo", "acapbot"],
|
|
275
|
+
["bitlybot", "seo", "bitlybot"],
|
|
276
|
+
["blogmuraBot", "seo", "blogmuraBot"],
|
|
277
|
+
["Bot.AraTurka.com", "search", "Bot\\.AraTurka\\.com"],
|
|
278
|
+
["bot-pge.chlooe.com", "seo", "bot-pge\\.chlooe\\.com"],
|
|
279
|
+
["BoxcarBot", "seo", "BoxcarBot"],
|
|
280
|
+
["Digincore bot", "seo", "Digincore bot"],
|
|
281
|
+
["Disqus", "seo", "Disqus"],
|
|
282
|
+
["Fetch", "seo", "Fetch\\/"],
|
|
283
|
+
["Flamingo_SearchEngine", "search", "Flamingo_SearchEngine"],
|
|
284
|
+
["G2 Web Services", "seo", "G2 Web Services"],
|
|
285
|
+
["imrbot", "seo", "imrbot"],
|
|
286
|
+
["Kemvibot", "seo", "Kemvibot"],
|
|
287
|
+
["Landau-Media-Spider", "seo", "Landau-Media-Spider"],
|
|
288
|
+
["linkapediabot", "seo", "linkapediabot"],
|
|
289
|
+
["BLEXBot", "seo", "BLEXBot\\/"],
|
|
290
|
+
["ZuperlistBot", "seo", "ZuperlistBot\\/"],
|
|
291
|
+
["Diffbot", "seo", "Diffbot\\/"],
|
|
292
|
+
["SEOkicks", "seo", "SEOkicks"],
|
|
293
|
+
["tracemyfile", "seo", "tracemyfile"],
|
|
294
|
+
["Nimbostratus-Bot", "seo", "Nimbostratus-Bot"],
|
|
295
|
+
["PR-CY.RU", "seo", "PR-CY\\.RU"],
|
|
296
|
+
["Datafeedwatch", "seo", "Datafeedwatch"],
|
|
297
|
+
["TangibleeBot", "seo", "TangibleeBot"],
|
|
298
|
+
["google-xrawler", "search", "google-xrawler"],
|
|
299
|
+
["CloudFlare-Prefetch", "seo", "CloudFlare-Prefetch"],
|
|
300
|
+
["Google-Structured-Data-Testing-Tool", "search", "Google-Structured-Data-Testing-Tool"],
|
|
301
|
+
["WordupInfoSearch", "search", "WordupInfoSearch"],
|
|
302
|
+
["WebDataStats", "seo", "WebDataStats"],
|
|
303
|
+
["ZoomBot", "seo", "ZoomBot"],
|
|
304
|
+
["VelenPublicWebCrawler", "seo", "VelenPublicWebCrawler"],
|
|
305
|
+
["MoodleBot", "seo", "MoodleBot"],
|
|
306
|
+
["jpg-newsbot", "seo", "jpg-newsbot"],
|
|
307
|
+
["Google-PhysicalWeb", "search", "Google-PhysicalWeb"],
|
|
308
|
+
["ICBot", "seo", "ICBot\\/"],
|
|
309
|
+
["Twingly", "seo", "Twingly"],
|
|
310
|
+
["Rivva", "seo", "Rivva"],
|
|
311
|
+
["awesomecrawler", "seo", "awesomecrawler"],
|
|
312
|
+
["Dataprovider.com", "seo", "Dataprovider\\.com"],
|
|
313
|
+
["GroupHigh", "seo", "GroupHigh\\/"],
|
|
314
|
+
["2ip.ru", "seo", "2ip\\.ru"],
|
|
315
|
+
["Caliperbot", "seo", "Caliperbot"],
|
|
316
|
+
["MBCrawler", "seo", "MBCrawler"],
|
|
317
|
+
["online-webceo-bot", "seo", "online-webceo-bot"],
|
|
318
|
+
["B2B Bot", "seo", "B2B Bot"],
|
|
319
|
+
["AddSearchBot", "search", "AddSearchBot"],
|
|
320
|
+
["Google Favicon", "search", "Google Favicon"],
|
|
321
|
+
["HubSpot", "seo", "HubSpot"],
|
|
322
|
+
["serpstatbot", "seo", "serpstatbot\\/"],
|
|
323
|
+
["MixnodeCache", "seo", "MixnodeCache\\/"],
|
|
324
|
+
["SimpleScraper", "seo", "SimpleScraper"],
|
|
325
|
+
["Jooblebot", "search", "Jooblebot"],
|
|
326
|
+
["RegionStuttgartBot", "seo", "RegionStuttgartBot"],
|
|
327
|
+
["Datanyze", "seo", "Datanyze"],
|
|
328
|
+
["Google-Site-Verification", "search", "Google-Site-Verification"],
|
|
329
|
+
["TrendsmapResolver", "seo", "TrendsmapResolver"],
|
|
330
|
+
["tweetedtimes", "seo", "tweetedtimes"],
|
|
331
|
+
["NTENTbot", "search", "NTENTbot"],
|
|
332
|
+
["SearchAtlas", "seo", "SearchAtlas"],
|
|
333
|
+
["AmazonProductDiscovery", "search", "AmazonProductDiscovery"],
|
|
334
|
+
["AmazonSellerInitiatedListing", "search", "AmazonSellerInitiatedListing"],
|
|
335
|
+
["SerendeputyBot", "seo", "SerendeputyBot"],
|
|
336
|
+
["officestorebot", "search", "officestorebot"],
|
|
337
|
+
["Neticle Crawler", "seo", "Neticle Crawler"],
|
|
338
|
+
["SurdotlyBot", "seo", "SurdotlyBot"],
|
|
339
|
+
["LinkisBot", "seo", "LinkisBot"],
|
|
340
|
+
["AwarioSmartBot", "seo", "AwarioSmartBot"],
|
|
341
|
+
["AwarioRssBot", "seo", "AwarioRssBot"],
|
|
342
|
+
["RyteBot", "seo", "RyteBot"],
|
|
343
|
+
["AspiegelBot", "search", "AspiegelBot"],
|
|
344
|
+
["zenback bot", "seo", "zenback bot"],
|
|
345
|
+
["SentiBot", "seo", "SentiBot"],
|
|
346
|
+
["Domains Project", "seo", "Domains Project\\/"],
|
|
347
|
+
["Pandalytics", "seo", "Pandalytics"],
|
|
348
|
+
["tigerbot", "seo", "tigerbot"],
|
|
349
|
+
["PagePeeker", "seo", "PagePeeker\\/"],
|
|
350
|
+
["startmebot", "seo", "startmebot\\/"],
|
|
351
|
+
["JobboerseBot", "search", "JobboerseBot"],
|
|
352
|
+
["seewithkids", "seo", "seewithkids"],
|
|
353
|
+
["Cutbot", "seo", "Cutbot"],
|
|
354
|
+
["BublupBot", "seo", "BublupBot"],
|
|
355
|
+
["BrandONbot", "seo", "BrandONbot"],
|
|
356
|
+
["RidderBot", "seo", "RidderBot"],
|
|
357
|
+
["FindITAnswersbot", "seo", "FindITAnswersbot"],
|
|
358
|
+
["infoobot", "seo", "infoobot"],
|
|
359
|
+
["Refindbot", "seo", "Refindbot"],
|
|
360
|
+
["SeobilityBot", "seo", "SeobilityBot"],
|
|
361
|
+
["Cincraw", "seo", "Cincraw"],
|
|
362
|
+
["Dragonbot", "seo", "Dragonbot"],
|
|
363
|
+
["BitBot", "seo", "BitBot"],
|
|
364
|
+
["Google-Certificates-Bridge", "search", "Google-Certificates-Bridge"],
|
|
365
|
+
["centurybot", "seo", "centurybot"],
|
|
366
|
+
["e.ventures Investment Crawler", "seo", "e\\.ventures Investment Crawler"],
|
|
367
|
+
["evc-batch", "seo", "evc-batch"],
|
|
368
|
+
["PetalBot", "search", "PetalBot"],
|
|
369
|
+
["trovitBot", "search", "trovitBot"],
|
|
370
|
+
["seostar.co", "seo", "seostar\\.co"],
|
|
371
|
+
["IonCrawl", "seo", "IonCrawl"],
|
|
372
|
+
["Seekport", "search", "Seekport"],
|
|
373
|
+
["YandexRenderResourcesBot", "search", "YandexRenderResourcesBot\\/"],
|
|
374
|
+
["ev-crawler", "seo", "ev-crawler\\/"],
|
|
375
|
+
["woorankreview", "seo", "woorankreview\\/"],
|
|
376
|
+
["Google-Safety", "search", "Google-Safety"],
|
|
377
|
+
["AwarioBot", "seo", "AwarioBot"],
|
|
378
|
+
["DataForSeoBot", "seo", "DataForSeoBot"],
|
|
379
|
+
["WellKnownBot", "seo", "WellKnownBot"],
|
|
380
|
+
["StractBot", "search", "StractBot"],
|
|
381
|
+
["search.marginalia.nu", "search", "search\\.marginalia\\.nu"],
|
|
382
|
+
["Nicecrawler", "search", "Nicecrawler"],
|
|
383
|
+
["Neevabot", "search", "Neevabot"],
|
|
384
|
+
["BrightEdge Crawler", "seo", "BrightEdge Crawler"],
|
|
385
|
+
["SiteCheckerBotCrawler", "seo", "SiteCheckerBotCrawler"],
|
|
386
|
+
["TombaPublicWebCrawler", "seo", "TombaPublicWebCrawler"],
|
|
387
|
+
["CrawlyProjectCrawler", "seo", "CrawlyProjectCrawler"],
|
|
388
|
+
["KomodiaBot", "seo", "KomodiaBot"],
|
|
389
|
+
["KStandBot", "seo", "KStandBot"],
|
|
390
|
+
["MTRobot", "seo", "MTRobot"],
|
|
391
|
+
["hyscore.io", "seo", "hyscore\\.io"],
|
|
392
|
+
["AlexandriaOrgBot", "search", "AlexandriaOrgBot"],
|
|
393
|
+
["2ip bot", "seo", "2ip bot"],
|
|
394
|
+
["Yellowbrandprotectionbot", "seo", "Yellowbrandprotectionbot"],
|
|
395
|
+
["SEOlizer", "seo", "SEOlizer"],
|
|
396
|
+
["vuhuvBot", "seo", "vuhuvBot"],
|
|
397
|
+
["INETDEX-BOT", "search", "INETDEX-BOT"],
|
|
398
|
+
["t3versionsBot", "seo", "t3versionsBot"],
|
|
399
|
+
["Cocolyzebot", "seo", "Cocolyzebot"],
|
|
400
|
+
["hypestat", "seo", "hypestat"],
|
|
401
|
+
["sempi.tech", "seo", "sempi\\.tech"],
|
|
402
|
+
["MetaInspector", "seo", "MetaInspector"],
|
|
403
|
+
["OpenGraphCheck", "seo", "OpenGraphCheck"],
|
|
404
|
+
["developers.google.com/+/web/snippet", "search", "developers\\.google\\.com\\/\\+\\/web\\/snippet"],
|
|
405
|
+
["SenutoBot", "seo", "SenutoBot"],
|
|
406
|
+
["PageThing", "seo", "PageThing"],
|
|
407
|
+
["WordPress", "seo", "WordPress\\/"],
|
|
408
|
+
["PhxBot", "search", "PhxBot"],
|
|
409
|
+
["ImagesiftBot", "search", "ImagesiftBot"],
|
|
410
|
+
["BW", "seo", "^BW\\/"],
|
|
411
|
+
["GeedoBot", "search", "GeedoBot"],
|
|
412
|
+
["Audisto Crawler", "seo", "Audisto Crawler"],
|
|
413
|
+
["Vercelbot", "seo", "Vercelbot"],
|
|
414
|
+
["vercel-screenshot", "seo", "vercel-screenshot"],
|
|
415
|
+
["GeedoProductSearch", "search", "GeedoProductSearch"],
|
|
416
|
+
["OnCrawl", "seo", "OnCrawl\\/"],
|
|
417
|
+
["sitebulb", "seo", "sitebulb"],
|
|
418
|
+
["YextBot", "seo", "YextBot\\/"],
|
|
419
|
+
["Timpibot", "search", "Timpibot"],
|
|
420
|
+
["SERankingBacklinksBot", "seo", "SERankingBacklinksBot"],
|
|
421
|
+
["CMSChecker", "seo", "CMSChecker"],
|
|
422
|
+
["OICrawler", "search", "OICrawler"],
|
|
423
|
+
["SlaccaleBot", "seo", "SlaccaleBot"],
|
|
424
|
+
["Gemini-Deep-Research", "search", "Gemini-Deep-Research"],
|
|
425
|
+
["PerplexityUser", "search", "PerplexityUser"],
|
|
426
|
+
["webzio", "seo", "webzio"],
|
|
427
|
+
["Attracta", "seo", "Attracta"],
|
|
428
|
+
["Cloudflare-Custom-Hostname-Verification", "seo", "Cloudflare-Custom-Hostname-Verification"],
|
|
429
|
+
["Cloudflare-Stream-Webhook", "seo", "Cloudflare-Stream-Webhook"],
|
|
430
|
+
["Cloudtrellis", "seo", "Cloudtrellis"],
|
|
431
|
+
["Convermax", "seo", "Convermax"],
|
|
432
|
+
["DeepCrawl", "seo", "DeepCrawl"],
|
|
433
|
+
["FindFiles.net", "search", "FindFiles\\.net"],
|
|
434
|
+
["FyndSearchEngine-Crawler", "search", "FyndSearchEngine-Crawler"],
|
|
435
|
+
["FyndSearchEngine-ReCrawler", "search", "FyndSearchEngine-ReCrawler"],
|
|
436
|
+
["Greppr Web Crawler", "search", "Greppr Web Crawler"],
|
|
437
|
+
["IbouBot", "search", "IbouBot"],
|
|
438
|
+
["LinkCheckerBot", "seo", "LinkCheckerBot"],
|
|
439
|
+
["lyonl-asset-proxy", "search", "lyonl-asset-proxy"],
|
|
440
|
+
["lyonl-crawler", "search", "lyonl-crawler"],
|
|
441
|
+
["MagiBot", "search", "MagiBot"],
|
|
442
|
+
["ModernizeBot", "seo", "ModernizeBot"],
|
|
443
|
+
["NestDaddybot", "search", "NestDaddybot"],
|
|
444
|
+
["Nitro", "seo", "Nitro-"],
|
|
445
|
+
["NitroBot", "seo", "NitroBot"],
|
|
446
|
+
["Qwantbot", "search", "Qwantbot"],
|
|
447
|
+
["RSiteAuditor", "seo", "RSiteAuditor"],
|
|
448
|
+
["Scrubby", "search", "Scrubby"],
|
|
449
|
+
["seo4ajax.com", "seo", "seo4ajax\\.com"],
|
|
450
|
+
["SMTnetPMBot", "search", "SMTnetPMBot"],
|
|
451
|
+
["StatusNestBacklinkSpider", "seo", "StatusNestBacklinkSpider"],
|
|
452
|
+
["stepstoneCrawlBot", "search", "stepstoneCrawlBot"],
|
|
453
|
+
["WepchSearchEngine", "search", "WepchSearchEngine"],
|
|
454
|
+
["XoviOnpageCrawler", "seo", "XoviOnpageCrawler"],
|
|
455
|
+
["ZoomInfo", "search", "ZoomInfo-"],
|
|
456
|
+
["7Siters", "search", "7Siters"],
|
|
457
|
+
["CyberFindCrawler", "search", "CyberFindCrawler"],
|
|
458
|
+
["FediIndex", "search", "FediIndex"],
|
|
459
|
+
["fiperbot", "search", "fiperbot"],
|
|
460
|
+
["FleebsBot", "search", "FleebsBot"],
|
|
461
|
+
["MatchorySearch", "search", "MatchorySearch"],
|
|
462
|
+
["NapBot", "search", "NapBot"],
|
|
463
|
+
["seoLyt", "seo", "seoLyt"],
|
|
464
|
+
["SitesOverPagesBot", "seo", "SitesOverPagesBot"],
|
|
465
|
+
["Sosospider", "search", "Sosospider"],
|
|
466
|
+
["WadooBot", "search", "WadooBot"],
|
|
467
|
+
["Website-info.net-Robot", "seo", "Website-info\\.net-Robot"],
|
|
468
|
+
["WikiDo", "search", "WikiDo"],
|
|
469
|
+
["YoudaoBot", "search", "YoudaoBot"],
|
|
470
|
+
["ZyBorg", "search", "ZyBorg"],
|
|
471
|
+
["blekkobot", "search", "blekkobot"],
|
|
472
|
+
["br-crawler", "search", "br-crawler"],
|
|
473
|
+
["gsa-crawler", "search", "gsa-crawler"],
|
|
474
|
+
["NaverBot", "search", "NaverBot"],
|
|
475
|
+
["OpenTheBoxBot", "search", "OpenTheBoxBot"],
|
|
476
|
+
["Swiftbot", "search", "Swiftbot"],
|
|
477
|
+
["cognitiveseo.com", "seo", "cognitiveseo\\.com"],
|
|
478
|
+
["XML Sitemaps Generator", "seo", "XML Sitemaps Generator"],
|
|
479
|
+
["Zealbot", "search", "Zealbot"],
|
|
480
|
+
["008", "seo", "008\\/"],
|
|
481
|
+
["AdagioBot", "seo", "AdagioBot"],
|
|
482
|
+
["AdminLabs", "seo", "AdminLabs"],
|
|
483
|
+
["advanced_crawler", "seo", "advanced_crawler"],
|
|
484
|
+
["Adventurer", "seo", "Adventurer"],
|
|
485
|
+
["AGAKIDSBOT", "seo", "AGAKIDSBOT"],
|
|
486
|
+
["AgencyAnalyticsBot", "seo", "AgencyAnalyticsBot"],
|
|
487
|
+
["alexa site audit", "seo", "alexa site audit"],
|
|
488
|
+
["Algolia Crawler", "search", "Algolia Crawler"],
|
|
489
|
+
["alienfarm", "seo", "alienfarm"],
|
|
490
|
+
["allOrigins", "seo", "allOrigins"],
|
|
491
|
+
["KendraBot", "search", "KendraBot"],
|
|
492
|
+
["AppSiteAssociation", "seo", "AppSiteAssociation"],
|
|
493
|
+
["Aragog", "seo", "Aragog\\/"],
|
|
494
|
+
["Asana", "seo", "Asana\\/"],
|
|
495
|
+
["AudistoBot", "seo", "AudistoBot"],
|
|
496
|
+
["BDBot", "seo", "BDBot\\/"],
|
|
497
|
+
["BeeperBot", "seo", "BeeperBot"],
|
|
498
|
+
["BigUpDataBot", "seo", "BigUpDataBot"],
|
|
499
|
+
["Bitbucket-Webhooks", "seo", "Bitbucket-Webhooks"],
|
|
500
|
+
["BoardGamePricesBot", "seo", "BoardGamePricesBot"],
|
|
501
|
+
["BotPoke", "seo", "BotPoke"],
|
|
502
|
+
["BDFetch", "seo", "BDFetch"],
|
|
503
|
+
["Brandwatch", "seo", "Brandwatch"],
|
|
504
|
+
["BraveBot", "search", "BraveBot"],
|
|
505
|
+
["brokenlinkcheck.com", "seo", "brokenlinkcheck\\.com"],
|
|
506
|
+
["BW", "seo", "BW\\/"],
|
|
507
|
+
["Bushbaby", "seo", "Bushbaby"],
|
|
508
|
+
["Butterfly", "seo", "Butterfly"],
|
|
509
|
+
["CaliberBot", "seo", "CaliberBot"],
|
|
510
|
+
["CapitalOneShopping", "seo", "CapitalOneShopping"],
|
|
511
|
+
["centuryb.o.t9", "seo", "centuryb\\.o\\.t9"],
|
|
512
|
+
["certytags", "seo", "certytags"],
|
|
513
|
+
["ChargeBeeBot", "seo", "ChargeBeeBot"],
|
|
514
|
+
["Charlotte", "seo", "Charlotte"],
|
|
515
|
+
["Goodzer", "seo", "Goodzer"],
|
|
516
|
+
["Chrome Privacy Preserving Prefetch Proxy", "seo", "Chrome Privacy Preserving Prefetch Proxy"],
|
|
517
|
+
["CirrusExplorer", "seo", "CirrusExplorer"],
|
|
518
|
+
["Clearscopebot", "seo", "Clearscopebot"],
|
|
519
|
+
["WorldBot", "seo", "WorldBot"],
|
|
520
|
+
["Cloudflare-Stream-Hook", "seo", "Cloudflare-Stream-Hook"],
|
|
521
|
+
["cognitiveSEO Bot", "seo", "cognitiveSEO Bot"],
|
|
522
|
+
["contentkingapp", "seo", "contentkingapp"],
|
|
523
|
+
["Cotoyogi", "seo", "Cotoyogi"],
|
|
524
|
+
["Coveobot", "search", "Coveobot"],
|
|
525
|
+
["Crawlson", "seo", "Crawlson"],
|
|
526
|
+
["RepoLookoutBot", "seo", "RepoLookoutBot"],
|
|
527
|
+
["Criticalcss.com", "seo", "Criticalcss\\.com"],
|
|
528
|
+
["DnBCrawler", "seo", "DnBCrawler"],
|
|
529
|
+
["DomCopBot", "seo", "DomCopBot"],
|
|
530
|
+
["Dlc", "seo", "Dlc\\/"],
|
|
531
|
+
["easybill-ImportManager", "seo", "easybill-ImportManager"],
|
|
532
|
+
["EchoboxBot", "seo", "EchoboxBot\\/"],
|
|
533
|
+
["crusty", "seo", "crusty\\/"],
|
|
534
|
+
["CXK_Bot", "seo", "CXK_Bot"],
|
|
535
|
+
["daumoa", "search", "daumoa"],
|
|
536
|
+
["DaspeedBot", "seo", "DaspeedBot"],
|
|
537
|
+
["Dead Link Checker", "seo", "Dead Link Checker"],
|
|
538
|
+
["Deskyobot", "seo", "Deskyobot"],
|
|
539
|
+
["DF Bot", "seo", "DF Bot"],
|
|
540
|
+
["Dmbot", "seo", "Dmbot"],
|
|
541
|
+
["SustainabilityCrawler", "seo", "SustainabilityCrawler"],
|
|
542
|
+
["edansbot", "seo", "edansbot"],
|
|
543
|
+
["eMoneyBot", "seo", "eMoneyBot"],
|
|
544
|
+
["EpivozCrawler", "seo", "EpivozCrawler"],
|
|
545
|
+
["eRepublik.tools", "seo", "eRepublik\\.tools"],
|
|
546
|
+
["ExodusMovement", "seo", "ExodusMovement"],
|
|
547
|
+
["Ezgif", "seo", "Ezgif"],
|
|
548
|
+
["factset_spyderbot", "seo", "factset_spyderbot"],
|
|
549
|
+
["FDL Stats Bot", "seo", "FDL Stats Bot"],
|
|
550
|
+
["Fedicabot", "seo", "Fedicabot"],
|
|
551
|
+
["FedReporterDataBot", "seo", "FedReporterDataBot"],
|
|
552
|
+
["fidget-spinner-bot", "seo", "fidget-spinner-bot"],
|
|
553
|
+
["FirmoGraph", "seo", "FirmoGraph"],
|
|
554
|
+
["Freespoke", "search", "Freespoke\\/"],
|
|
555
|
+
["Friendly testing bot", "seo", "Friendly testing bot"],
|
|
556
|
+
["friendly-spider", "seo", "friendly-spider"],
|
|
557
|
+
["FriendlyCrawler", "seo", "FriendlyCrawler\\/"],
|
|
558
|
+
["FullStoryBot", "seo", "FullStoryBot\\/"],
|
|
559
|
+
["Funnelback", "search", "Funnelback"],
|
|
560
|
+
["FuseonBot", "seo", "FuseonBot\\/"],
|
|
561
|
+
["Gabanzabot", "search", "Gabanzabot\\/"],
|
|
562
|
+
["gdnplus.com", "seo", "gdnplus\\.com"],
|
|
563
|
+
["getthit.com", "seo", "getthit\\.com"],
|
|
564
|
+
["GlobalWebSearch", "search", "GlobalWebSearch"],
|
|
565
|
+
["Golfe", "search", "Golfe\\/"],
|
|
566
|
+
["Google-Apps-Script", "seo", "Google-Apps-Script"],
|
|
567
|
+
["GoogleAssociationService", "seo", "GoogleAssociationService\\/"],
|
|
568
|
+
["GoogleProducer", "search", "GoogleProducer"],
|
|
569
|
+
["Google-Area120", "seo", "Google-Area120"],
|
|
570
|
+
["GoogleAssociationService", "seo", "GoogleAssociationService$"],
|
|
571
|
+
["GoogleDocs", "seo", "GoogleDocs"],
|
|
572
|
+
["GoPay", "seo", "GoPay"],
|
|
573
|
+
["Grammarly", "seo", "Grammarly\\/"],
|
|
574
|
+
["gregcrawler", "seo", "gregcrawler"],
|
|
575
|
+
["Grover", "seo", "Grover\\/"],
|
|
576
|
+
["GuestpostsBot", "seo", "GuestpostsBot\\/"],
|
|
577
|
+
["HappyWing", "seo", "HappyWing"],
|
|
578
|
+
["harsilbot", "seo", "harsilbot\\/"],
|
|
579
|
+
["HawaiiBot", "seo", "HawaiiBot"],
|
|
580
|
+
["hCardValidator", "seo", "hCardValidator"],
|
|
581
|
+
["Hello World", "seo", "Hello World"],
|
|
582
|
+
["HelloworkJobPostingBot", "seo", "HelloworkJobPostingBot\\/"],
|
|
583
|
+
["HIFIBot", "seo", "HIFIBot\\/"],
|
|
584
|
+
["Hlidam.to robot", "seo", "Hlidam\\.to robot"],
|
|
585
|
+
["Hotjar", "seo", "Hotjar"],
|
|
586
|
+
["Huckabot", "seo", "Huckabot\\/"],
|
|
587
|
+
["Hype Machine", "seo", "Hype Machine\\/"],
|
|
588
|
+
["Web Screen Service By hyperhost", "seo", "Web Screen Service By hyperhost"],
|
|
589
|
+
["iAskBot", "search", "iAskBot\\/"],
|
|
590
|
+
["IBM Crawler", "seo", "IBM Crawler"],
|
|
591
|
+
["IFTTT", "seo", "IFTTT\\/"],
|
|
592
|
+
["ImageFetcher", "seo", "ImageFetcher\\/"],
|
|
593
|
+
["imgproxy", "seo", "imgproxy\\/"],
|
|
594
|
+
["impendoom-bot", "seo", "impendoom-bot\\/"],
|
|
595
|
+
["IndeedJobBot", "search", "IndeedJobBot"],
|
|
596
|
+
["Innguma", "seo", "Innguma\\/"],
|
|
597
|
+
["Instapaper", "seo", "Instapaper\\/"],
|
|
598
|
+
["Integromat", "seo", "Integromat\\/"],
|
|
599
|
+
["intelx.io_bot", "search", "intelx\\.io_bot"],
|
|
600
|
+
["jobswithgptcom-bot", "search", "jobswithgptcom-bot"],
|
|
601
|
+
["Jumio", "seo", "Jumio"],
|
|
602
|
+
["Kagibot", "search", "Kagibot\\/"],
|
|
603
|
+
["KangarooBot", "seo", "KangarooBot\\/"],
|
|
604
|
+
["kazbtbot", "seo", "kazbtbot\\/"],
|
|
605
|
+
["keys-so-bot", "seo", "keys-so-bot"],
|
|
606
|
+
["kinsta-bot", "seo", "kinsta-bot"],
|
|
607
|
+
["Klaviyo", "seo", "Klaviyo\\/"],
|
|
608
|
+
["Kukei.eu-Bot", "seo", "Kukei\\.eu-Bot\\/"],
|
|
609
|
+
["LastModBot", "seo", "LastModBot\\/"],
|
|
610
|
+
["Level9SearchBot", "search", "Level9SearchBot\\/"],
|
|
611
|
+
["LinkTiger", "seo", "LinkTiger"],
|
|
612
|
+
["LinksIndexerBot", "seo", "LinksIndexerBot\\/"],
|
|
613
|
+
["LinkWalker", "seo", "LinkWalker\\/"],
|
|
614
|
+
["Macrobondbot", "seo", "Macrobondbot"],
|
|
615
|
+
["MADBbot", "seo", "MADBbot\\/"],
|
|
616
|
+
["Magellan", "seo", "Magellan"],
|
|
617
|
+
["magicsearchdev", "seo", "magicsearchdev\\/"],
|
|
618
|
+
["Magnet.me-web", "seo", "Magnet\\.me-web\\/"],
|
|
619
|
+
["MainWP", "seo", "MainWP\\/"],
|
|
620
|
+
["Make", "seo", "Make\\/"],
|
|
621
|
+
["ManageWP", "seo", "ManageWP"],
|
|
622
|
+
["MarketGoo", "seo", "MarketGoo\\/"],
|
|
623
|
+
["MarketingMiner", "seo", "MarketingMiner"],
|
|
624
|
+
["Mavifinds", "seo", "Mavifinds"],
|
|
625
|
+
["MB-LinkChecker", "seo", "MB-LinkChecker"],
|
|
626
|
+
["MedialogiaBot", "seo", "MedialogiaBot"],
|
|
627
|
+
["MediaMonitoringBot", "seo", "MediaMonitoringBot\\/"],
|
|
628
|
+
["Pywikibot", "seo", "Pywikibot\\/"],
|
|
629
|
+
["CentComBot", "seo", "CentComBot\\/"],
|
|
630
|
+
["MergadoBot", "seo", "MergadoBot"],
|
|
631
|
+
["Metorik", "seo", "Metorik"],
|
|
632
|
+
["MissinglettrBot", "seo", "MissinglettrBot\\/"],
|
|
633
|
+
["crawler_eb_germany", "seo", "crawler_eb_germany"],
|
|
634
|
+
["ModularConnector", "seo", "ModularConnector\\/"],
|
|
635
|
+
["Mollie HTTP client", "seo", "Mollie HTTP client"],
|
|
636
|
+
["Monibot", "seo", "Monibot"],
|
|
637
|
+
["MonTools.com", "seo", "MonTools\\.com"],
|
|
638
|
+
["MotoMinerBot", "seo", "MotoMinerBot\\/"],
|
|
639
|
+
["my-tiny-bot", "seo", "my-tiny-bot"],
|
|
640
|
+
["MyBot", "seo", "MyBot\\/"],
|
|
641
|
+
["nbertaupete95", "seo", "nbertaupete95"],
|
|
642
|
+
["NetAPI", "seo", "NetAPI"],
|
|
643
|
+
["NetpeakCheckerBot", "seo", "NetpeakCheckerBot\\/"],
|
|
644
|
+
["nyt_scraping", "seo", "nyt_scraping"],
|
|
645
|
+
["NewsNow", "seo", "NewsNow\\/"],
|
|
646
|
+
["nomore404.com robot", "seo", "nomore404\\.com robot"],
|
|
647
|
+
["noorobot", "seo", "noorobot"],
|
|
648
|
+
["Nooshub", "seo", "Nooshub\\/"],
|
|
649
|
+
["Notabot", "seo", "Notabot"],
|
|
650
|
+
["Novaact", "seo", "Novaact\\/"],
|
|
651
|
+
["nvdorz", "seo", "nvdorz"],
|
|
652
|
+
["Odin", "seo", "Odin;"],
|
|
653
|
+
["Omnisend", "seo", "Omnisend\\/"],
|
|
654
|
+
["Online Domain Tools", "seo", "Online Domain Tools"],
|
|
655
|
+
["WebCEO Online", "seo", "WebCEO Online\\/"],
|
|
656
|
+
["Owler (ows.eu", "search", "Owler \\(ows\\.eu"],
|
|
657
|
+
["Operator", "seo", "Operator\\/"],
|
|
658
|
+
["Orbbot", "seo", "Orbbot\\/"],
|
|
659
|
+
["zebra-v2-bot", "seo", "zebra-v2-bot"],
|
|
660
|
+
["OutsellURLValidator", "seo", "OutsellURLValidator"],
|
|
661
|
+
["PanguBot", "search", "PanguBot"],
|
|
662
|
+
["Paqlebot", "seo", "Paqlebot\\/"],
|
|
663
|
+
["parse.ly scraper", "seo", "parse\\.ly scraper\\/"],
|
|
664
|
+
["PayPal", "seo", "PayPal\\/"],
|
|
665
|
+
["PDF24 URL To PDF", "seo", "PDF24 URL To PDF"],
|
|
666
|
+
["PlayStore-Google", "search", "PlayStore-Google"],
|
|
667
|
+
["Plesk screenshot bot", "seo", "Plesk screenshot bot"],
|
|
668
|
+
["Potions", "seo", "Potions\\/"],
|
|
669
|
+
["PressEngineBot", "seo", "PressEngineBot"],
|
|
670
|
+
["PricedroneShoppingBot", "seo", "PricedroneShoppingBot\\/"],
|
|
671
|
+
["PriEcoBot", "seo", "PriEcoBot\\/"],
|
|
672
|
+
["PrintFriendly.com", "seo", "PrintFriendly\\.com"],
|
|
673
|
+
["Pro-Sitemaps", "seo", "Pro-Sitemaps\\/"],
|
|
674
|
+
["PS_Daily", "seo", "PS_Daily\\/"],
|
|
675
|
+
["PWABuilderHttpAgent", "seo", "PWABuilderHttpAgent"],
|
|
676
|
+
["QualifiedBot", "seo", "QualifiedBot\\/"],
|
|
677
|
+
["rakutenusabot-image", "seo", "rakutenusabot-image\\/"],
|
|
678
|
+
["top100.rambler.ru crawler", "search", "top100\\.rambler\\.ru crawler"],
|
|
679
|
+
["RankurBot", "seo", "RankurBot\\/"],
|
|
680
|
+
["RavenCrawler", "seo", "RavenCrawler\\/"],
|
|
681
|
+
["Readable", "seo", "Readable\\/"],
|
|
682
|
+
["Recurly Webhooks", "seo", "Recurly Webhooks\\/"],
|
|
683
|
+
["Reelevant", "seo", "Reelevant\\/"],
|
|
684
|
+
["remove.bg", "seo", "remove\\.bg\\/"],
|
|
685
|
+
["Retool", "seo", "Retool\\/"],
|
|
686
|
+
["RetroListeCOM", "seo", "RetroListeCOM\\/"],
|
|
687
|
+
["RevvimGort", "seo", "RevvimGort\\/"],
|
|
688
|
+
["reward-gateway", "seo", "reward-gateway"],
|
|
689
|
+
["RobotsChecker", "seo", "RobotsChecker\\/"],
|
|
690
|
+
["s4a-probe-bot", "seo", "s4a-probe-bot\\/"],
|
|
691
|
+
["SFDC-Callout", "seo", "SFDC-Callout\\/"],
|
|
692
|
+
["page-preview-tool", "seo", "page-preview-tool"],
|
|
693
|
+
["SandobaCrawler", "seo", "SandobaCrawler\\/"],
|
|
694
|
+
["GIFTEDVISITOR SCAN", "seo", "GIFTEDVISITOR SCAN"],
|
|
695
|
+
["Schema-Markup-Validator", "seo", "Schema-Markup-Validator"],
|
|
696
|
+
["Scoop.it", "seo", "Scoop\\.it\\/"],
|
|
697
|
+
["ScrapeheroBot", "seo", "ScrapeheroBot\\/"],
|
|
698
|
+
["screeenly-bot", "seo", "screeenly-bot"],
|
|
699
|
+
["SEBot-WA", "seo", "SEBot-WA"],
|
|
700
|
+
["Searcherweb", "search", "Searcherweb"],
|
|
701
|
+
["Searcherxweb", "search", "Searcherxweb"],
|
|
702
|
+
["SearchExpress", "search", "SearchExpress"],
|
|
703
|
+
["semaltbot", "seo", "semaltbot\\/"],
|
|
704
|
+
["SendGrid Event API", "seo", "SendGrid Event API"],
|
|
705
|
+
["seo-audit-check-bot", "seo", "seo-audit-check-bot\\/"],
|
|
706
|
+
["s4a", "seo", "s4a\\/"],
|
|
707
|
+
["ClarityBot", "seo", "ClarityBot\\/"],
|
|
708
|
+
["SeoSiteCheckup", "seo", "SeoSiteCheckup"],
|
|
709
|
+
["SeoulBot", "seo", "SeoulBot"],
|
|
710
|
+
["SERPtimizerBot", "seo", "SERPtimizerBot"],
|
|
711
|
+
["ServerHunterSpider", "seo", "ServerHunterSpider\\/"],
|
|
712
|
+
["SeznamHomepageCrawler", "search", "SeznamHomepageCrawler\\/"],
|
|
713
|
+
["Shopify-Captain-Hook", "seo", "Shopify-Captain-Hook"],
|
|
714
|
+
["Sidetrade indexer bot", "seo", "Sidetrade indexer bot"],
|
|
715
|
+
["Silk", "seo", "^Silk\\/"],
|
|
716
|
+
["SinceraSyntheticUser", "seo", "SinceraSyntheticUser\\/"],
|
|
717
|
+
["Optimizer", "seo", "Optimizer\\)"],
|
|
718
|
+
["SiteAuditBot", "seo", "SiteAuditBot\\/"],
|
|
719
|
+
["SiteCheck-sitecrawl", "seo", "SiteCheck-sitecrawl"],
|
|
720
|
+
["SiteScoreBot", "seo", "SiteScoreBot"],
|
|
721
|
+
["SiteSearch360", "search", "SiteSearch360\\/"],
|
|
722
|
+
["Konturbot", "seo", "Konturbot\\/"],
|
|
723
|
+
["SkroutzBot", "seo", "SkroutzBot"],
|
|
724
|
+
["SkyworkSpider", "seo", "SkyworkSpider"],
|
|
725
|
+
["SlickBot", "seo", "SlickBot\\/"],
|
|
726
|
+
["Snipcart", "seo", "Snipcart\\/"],
|
|
727
|
+
["Sora POS", "seo", "Sora POS\\/"],
|
|
728
|
+
["SparkShipping", "seo", "SparkShipping"],
|
|
729
|
+
["SparkPost", "seo", "SparkPost"],
|
|
730
|
+
["IDG/EU", "seo", "IDG\\/EU"],
|
|
731
|
+
["SSSSBot", "seo", "SSSSBot\\/"],
|
|
732
|
+
["Stape", "seo", "Stape\\/"],
|
|
733
|
+
["StartpagePrivateImageProxy", "search", "StartpagePrivateImageProxy\\/"],
|
|
734
|
+
["Statabot", "seo", "Statabot\\/"],
|
|
735
|
+
["StatistikAustria", "seo", "StatistikAustria\\/"],
|
|
736
|
+
["StatsDroneBot", "seo", "StatsDroneBot"],
|
|
737
|
+
["Stripe", "seo", "Stripe\\/"],
|
|
738
|
+
["Svix-Webhooks", "seo", "Svix-Webhooks\\/"],
|
|
739
|
+
["SwifteqLinkChecker", "seo", "SwifteqLinkChecker"],
|
|
740
|
+
["Swisscows", "search", "Swisscows"],
|
|
741
|
+
["TactiScout", "seo", "TactiScout\\/"],
|
|
742
|
+
["tchelebi", "seo", "tchelebi\\/"],
|
|
743
|
+
["Testcrawler", "seo", "Testcrawler"],
|
|
744
|
+
["test-bot", "seo", "test-bot"],
|
|
745
|
+
["TestURI", "seo", "TestURI"],
|
|
746
|
+
["TextRazor", "seo", "TextRazor"],
|
|
747
|
+
["TheInternetSearchx", "search", "TheInternetSearchx"],
|
|
748
|
+
["Trellis-Services", "seo", "Trellis-Services"],
|
|
749
|
+
["trentwil.es", "seo", "trentwil\\.es"],
|
|
750
|
+
["Trustly", "seo", "Trustly\\/"],
|
|
751
|
+
["Tweakers", "seo", "Tweakers"],
|
|
752
|
+
["UASlinkChecker", "seo", "UASlinkChecker\\/"],
|
|
753
|
+
["Urlcheckr", "seo", "Urlcheckr\\/"],
|
|
754
|
+
["URLSuMaBot", "seo", "URLSuMaBot"],
|
|
755
|
+
["useeBookChecker", "seo", "useeBookChecker\\/"],
|
|
756
|
+
["Vagabondo", "seo", "Vagabondo\\/"],
|
|
757
|
+
["videootvBot", "seo", "videootvBot"],
|
|
758
|
+
["VsuSearchSpider", "search", "VsuSearchSpider\\/"],
|
|
759
|
+
["WARDBot", "seo", "WARDBot\\/"],
|
|
760
|
+
["Webflow", "seo", "Webflow"],
|
|
761
|
+
["webprosbot", "seo", "webprosbot\\/"],
|
|
762
|
+
["WebSniffer", "seo", "WebSniffer\\/"],
|
|
763
|
+
["WSM", "seo", "WSM\\/"],
|
|
764
|
+
["WebwikiBot", "seo", "WebwikiBot\\/"],
|
|
765
|
+
["WhatsMyIP.org", "seo", "WhatsMyIP\\.org"],
|
|
766
|
+
["Wheregoes.com", "seo", "Wheregoes\\.com"],
|
|
767
|
+
["WireReaderBot", "seo", "WireReaderBot\\/"],
|
|
768
|
+
["wmtips.com", "seo", "wmtips\\.com\\/"],
|
|
769
|
+
["WordCountBot", "seo", "WordCountBot\\/"],
|
|
770
|
+
["Wordup-1", "seo", "Wordup-1"],
|
|
771
|
+
["workona-favicon-service", "seo", "workona-favicon-service\\/"],
|
|
772
|
+
["Indy Library", "seo", "Indy Library"],
|
|
773
|
+
["WJHRO", "seo", "WJHRO\\/"],
|
|
774
|
+
["WovnCrawler", "seo", "WovnCrawler\\/"],
|
|
775
|
+
["wowLink Crawler", "seo", "wowLink Crawler\\/"],
|
|
776
|
+
["WP Time Capsule", "seo", "WP Time Capsule"],
|
|
777
|
+
["wpbot", "seo", "wpbot\\/"],
|
|
778
|
+
["WPMU DEV Broken Link Checker", "seo", "WPMU DEV Broken Link Checker"],
|
|
779
|
+
["WRTNBot", "seo", "WRTNBot"],
|
|
780
|
+
["YahooCacheSystem", "seo", "YahooCacheSystem"],
|
|
781
|
+
["YokoyGroupAG", "seo", "YokoyGroupAG\\/"],
|
|
782
|
+
["Yuuperbot", "seo", "Yuuperbot"],
|
|
783
|
+
["Zapier", "seo", "Zapier"],
|
|
784
|
+
["Zendesk Webhook", "seo", "Zendesk Webhook"],
|
|
785
|
+
["Zombiebot", "seo", "Zombiebot\\/"],
|
|
786
|
+
["zzhbot", "seo", "zzhbot"],
|
|
787
|
+
["Penthouse Critical Path CSS Generator", "seo", "Penthouse Critical Path CSS Generator"],
|
|
788
|
+
["Skroutz ImageBot", "seo", "Skroutz ImageBot"],
|
|
789
|
+
["GeedoShopProductFinder", "search", "GeedoShopProductFinder"],
|
|
790
|
+
["IsonSearchBot", "search", "IsonSearchBot"],
|
|
791
|
+
];
|