@pug-sh/crawlers 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,791 @@
1
+ // AUTOGENERATED: do not edit. Run `make codegen` to regenerate.
2
+ // table: pug's crawler table (proto/common/crawlers/v1), as [name, category, ...lowercased tokens].
3
+ // listed: crawler-user-agents@1.66.0, its search-engine and seo entries, as [name, category, pattern],
4
+ // under its license:
5
+ //
6
+ // The MIT License (MIT)
7
+ //
8
+ // Copyright (c) 2017 Martin Monperrus
9
+ //
10
+ // Permission is hereby granted, free of charge, to any person obtaining a copy of
11
+ // this software and associated documentation files (the "Software"), to deal in
12
+ // the Software without restriction, including without limitation the rights to
13
+ // use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
14
+ // the Software, and to permit persons to whom the Software is furnished to do so,
15
+ // subject to the following conditions:
16
+ //
17
+ // The above copyright notice and this permission notice shall be included in all
18
+ // copies or substantial portions of the Software.
19
+ //
20
+ // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21
+ // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
22
+ // FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
23
+ // COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
24
+ // IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
25
+ // CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26
+ export const table = [
27
+ ["ChatGPT-User", "answers", "chatgpt-user"],
28
+ ["Claude-User", "answers", "claude-user"],
29
+ ["Perplexity-User", "answers", "perplexity-user"],
30
+ ["MistralAI-User", "answers", "mistralai-user"],
31
+ ["Kimi-User", "answers", "kimi-user"],
32
+ ["DuckAssistBot", "answers", "duckassistbot"],
33
+ ["Amzn-User", "answers", "amzn-user"],
34
+ ["meta-externalfetcher", "answers", "meta-externalfetcher"],
35
+ ["Google-Agent", "answers", "google-agent"],
36
+ ["Google-GeminiNotebook", "answers", "google-gemininotebook", "google-notebooklm"],
37
+ ["Google-Read-Aloud", "answers", "google-read-aloud"],
38
+ ["OAI-SearchBot", "search", "oai-searchbot"],
39
+ ["Claude-SearchBot", "search", "claude-searchbot"],
40
+ ["PerplexityBot", "search", "perplexitybot"],
41
+ ["MistralAI-Index", "search", "mistralai-index"],
42
+ ["Kimi-SearchBot", "search", "kimi-searchbot"],
43
+ ["Amzn-SearchBot", "search", "amzn-searchbot"],
44
+ ["meta-webindexer", "search", "meta-webindexer"],
45
+ ["Googlebot", "search", "googlebot"],
46
+ ["bingbot", "search", "bingbot", "msnbot"],
47
+ ["Applebot", "search", "applebot"],
48
+ ["Baiduspider", "search", "baiduspider"],
49
+ ["YouBot", "search", "youbot"],
50
+ ["GPTBot", "training", "gptbot"],
51
+ ["ClaudeBot", "training", "claudebot"],
52
+ ["meta-externalagent", "training", "meta-externalagent"],
53
+ ["Amazonbot", "training", "amazonbot"],
54
+ ["KimiBot", "training", "kimibot"],
55
+ ["CCBot", "training", "ccbot"],
56
+ ["Bytespider", "training", "bytespider"],
57
+ ["AI2Bot", "training", "ai2bot"],
58
+ ["OAI-AdsBot", "other", "oai-adsbot"],
59
+ ["meta-externalads", "other", "meta-externalads"],
60
+ ["GoogleOther", "other", "googleother"],
61
+ ["Google-CloudVertexBot", "other", "google-cloudvertexbot"],
62
+ ];
63
+ export const listed = [
64
+ ["APIs-Google", "search", "APIs-Google"],
65
+ ["Google-InspectionTool", "search", "Google-InspectionTool"],
66
+ ["Storebot-Google", "search", "Storebot-Google"],
67
+ ["Slurp", "search", "Slurp"],
68
+ ["Nutch", "search", "Nutch"],
69
+ ["jyxobot", "search", "jyxobot"],
70
+ ["FAST-WebCrawler", "search", "FAST-WebCrawler"],
71
+ ["FAST Enterprise Crawler", "search", "FAST Enterprise Crawler"],
72
+ ["BIGLOTRON", "search", "BIGLOTRON"],
73
+ ["Teoma", "search", "Teoma"],
74
+ ["convera", "search", "convera"],
75
+ ["Seekbot", "search", "^Seekbot"],
76
+ ["Gigabot", "search", "Gigabot"],
77
+ ["Gigablast", "search", "Gigablast"],
78
+ ["exabot", "search", "exabot"],
79
+ ["GingerCrawler", "seo", "GingerCrawler"],
80
+ ["grub.org", "search", "grub\\.org"],
81
+ ["UsineNouvelleCrawler", "search", "UsineNouvelleCrawler"],
82
+ ["netresearchserver", "seo", "netresearchserver"],
83
+ ["speedy", "search", "speedy"],
84
+ ["fluffy", "search", "fluffy"],
85
+ ["findlink", "seo", "findlink"],
86
+ ["msrbot", "search", "msrbot"],
87
+ ["panscient", "seo", "panscient"],
88
+ ["yacybot", "search", "yacybot"],
89
+ ["AISearchBot", "search", "AISearchBot"],
90
+ ["ips-agent", "seo", "ips-agent"],
91
+ ["tagoobot", "search", "tagoobot"],
92
+ ["MJ12bot", "seo", "MJ12bot"],
93
+ ["woriobot", "search", "woriobot"],
94
+ ["yanga", "search", "yanga"],
95
+ ["buzzbot", "seo", "buzzbot"],
96
+ ["mlbot", "seo", "mlbot"],
97
+ ["yandex.com/bots", "search", "yandex\\.com\\/bots"],
98
+ ["purebot", "search", "purebot"],
99
+ ["Linguee Bot", "search", "Linguee Bot"],
100
+ ["voilabot", "search", "voilabot"],
101
+ ["spbot", "seo", "spbot"],
102
+ ["twengabot", "seo", "twengabot"],
103
+ ["postrank", "seo", "postrank"],
104
+ ["scribdbot", "search", "scribdbot"],
105
+ ["sitebot", "seo", "sitebot"],
106
+ ["linkdex", "seo", "linkdex"],
107
+ ["Adidxbot", "search", "Adidxbot"],
108
+ ["ezooms", "seo", "ezooms"],
109
+ ["dotbot", "seo", "dotbot"],
110
+ ["Mail.RU_Bot", "search", "Mail\\.RU_Bot"],
111
+ ["discobot", "search", "discobot"],
112
+ ["findthatfile", "search", "findthatfile"],
113
+ ["NerdByNature.Bot", "seo", "NerdByNature\\.Bot"],
114
+ ["SISTRIX Crawler", "seo", "(sistrix|SISTRIX) [cC]rawler"],
115
+ ["AhrefsBot", "seo", "Ahrefs(Bot|SiteAudit)"],
116
+ ["fuelbot", "seo", "fuelbot"],
117
+ ["CrunchBot", "seo", "^CrunchBot"],
118
+ ["IndeedBot", "search", "IndeedBot"],
119
+ ["mappydata", "seo", "mappydata"],
120
+ ["woobot", "seo", "woobot"],
121
+ ["ZoominfoBot", "seo", "ZoominfoBot"],
122
+ ["PrivacyAwareBot", "seo", "PrivacyAwareBot"],
123
+ ["Multiviewbot", "seo", "Multiviewbot"],
124
+ ["SWIMGBot", "seo", "SWIMGBot"],
125
+ ["Grobbot", "seo", "Grobbot"],
126
+ ["eright", "seo", "eright"],
127
+ ["Apercite", "seo", "Apercite"],
128
+ ["semanticbot", "seo", "semanticbot"],
129
+ ["Aboundex", "search", "Aboundex"],
130
+ ["domaincrawler", "seo", "domaincrawler"],
131
+ ["wbsearchbot", "search", "wbsearchbot"],
132
+ ["summify", "seo", "summify"],
133
+ ["edisterbot", "search", "edisterbot"],
134
+ ["SeznamBot", "search", "SeznamBot"],
135
+ ["gslfbot", "seo", "gslfbot"],
136
+ ["aiHitBot", "search", "aiHitBot"],
137
+ ["intelium_bot", "seo", "intelium_bot"],
138
+ ["Yeti", "search", "Yeti"],
139
+ ["RetrevoPageAnalyzer", "seo", "RetrevoPageAnalyzer"],
140
+ ["lb-spider", "search", "lb-spider"],
141
+ ["Sogou", "search", "Sogou"],
142
+ ["lssbot", "seo", "lssbot"],
143
+ ["careerbot", "seo", "careerbot"],
144
+ ["wotbox", "search", "wotbox"],
145
+ ["wocbot", "search", "wocbot"],
146
+ ["ichiro", "search", "ichiro"],
147
+ ["DuckDuckBot", "search", "DuckDuckBot"],
148
+ ["lssrocketcrawler", "seo", "lssrocketcrawler"],
149
+ ["drupact", "seo", "drupact"],
150
+ ["webcompanycrawler", "seo", "webcompanycrawler"],
151
+ ["acoonbot", "search", "acoonbot"],
152
+ ["openindexspider", "search", "openindexspider"],
153
+ ["gnam gnam spider", "seo", "gnam gnam spider"],
154
+ ["backlinkcrawler", "seo", "backlinkcrawler"],
155
+ ["coccoc", "search", "coccoc"],
156
+ ["content crawler spider", "seo", "content crawler spider"],
157
+ ["toplistbot", "seo", "toplistbot"],
158
+ ["it2media-domain-crawler", "seo", "it2media-domain-crawler"],
159
+ ["siteexplorer.info", "seo", "siteexplorer\\.info"],
160
+ ["elisabot", "seo", "elisabot"],
161
+ ["arabot", "search", "arabot"],
162
+ ["WeSEE:Search", "search", "WeSEE:Search"],
163
+ ["niki-bot", "seo", "niki-bot"],
164
+ ["CrystalSemanticsBot", "seo", "CrystalSemanticsBot"],
165
+ ["rogerbot", "seo", "rogerbot"],
166
+ ["360Spider", "search", "360Spider"],
167
+ ["psbot", "search", "psbot"],
168
+ ["InterfaxScanBot", "seo", "InterfaxScanBot"],
169
+ ["g00g1e.net", "seo", "g00g1e\\.net"],
170
+ ["urlappendbot", "seo", "urlappendbot"],
171
+ ["brainobot", "seo", "brainobot"],
172
+ ["fr-crawler", "search", "fr-crawler"],
173
+ ["smtbot", "seo", "smtbot"],
174
+ ["A6-Indexer", "search", "A6-Indexer"],
175
+ ["OrangeBot", "search", "OrangeBot\\/"],
176
+ ["memorybot", "seo", "memorybot"],
177
+ ["MegaIndex", "seo", "MegaIndex"],
178
+ ["nerdybot", "seo", "nerdybot"],
179
+ ["xovibot", "seo", "xovibot"],
180
+ ["Qwantify", "search", "Qwantify"],
181
+ ["findxbot", "search", "findxbot"],
182
+ ["SemrushBot", "seo", "S[eE][mM]rushBot"],
183
+ ["yoozBot", "seo", "yoozBot"],
184
+ ["lipperhey", "seo", "lipperhey"],
185
+ ["Y!J", "search", "Y!J"],
186
+ ["Domain Re-Animator Bot", "seo", "Domain Re-Animator Bot"],
187
+ ["Screaming Frog SEO Spider", "seo", "Screaming Frog SEO Spider"],
188
+ ["MetaURI", "seo", "MetaURI"],
189
+ ["LivelapBot", "seo", "Livelap[bB]ot"],
190
+ ["OpenHoseBot", "search", "OpenHoseBot"],
191
+ ["collection@infegy.com", "seo", "collection@infegy\\.com"],
192
+ ["IstellaBot", "search", "IstellaBot"],
193
+ ["DeuSu", "search", "DeuSu\\/"],
194
+ ["betaBot", "seo", "betaBot"],
195
+ ["Cliqzbot", "search", "Cliqzbot\\/"],
196
+ ["MojeekBot", "search", "MojeekBot\\/"],
197
+ ["netEstate NE Crawler", "seo", "netEstate NE Crawler"],
198
+ ["SafeSearch microdata crawler", "search", "SafeSearch microdata crawler"],
199
+ ["Gluten Free Crawler", "search", "Gluten Free Crawler\\/"],
200
+ ["Sonic", "seo", "Sonic"],
201
+ ["Sysomos", "seo", "Sysomos"],
202
+ ["deadlinkchecker", "seo", "deadlinkchecker"],
203
+ ["RankActiveLinkBot", "seo", "RankActiveLinkBot"],
204
+ ["iskanie", "seo", "iskanie"],
205
+ ["Veoozbot", "seo", "Veoozbot"],
206
+ ["datagnionbot", "seo", "datagnionbot"],
207
+ ["contxbot", "seo", "contxbot"],
208
+ ["electricmonk", "seo", "electricmonk"],
209
+ ["GarlikCrawler", "seo", "GarlikCrawler"],
210
+ ["vebidoobot", "seo", "vebidoobot"],
211
+ ["FemtosearchBot", "search", "FemtosearchBot"],
212
+ ["MetaJobBot", "seo", "MetaJobBot"],
213
+ ["DomainStatsBot", "seo", "DomainStatsBot"],
214
+ ["mindUpBot", "seo", "mindUpBot"],
215
+ ["Daum", "search", "Daum\\/"],
216
+ ["Xenu Link Sleuth", "seo", "Xenu Link Sleuth"],
217
+ ["KosmioBot", "seo", "KosmioBot"],
218
+ ["Gowikibot", "seo", "Gowikibot"],
219
+ ["PiplBot", "seo", "PiplBot"],
220
+ ["James BOT", "seo", "James BOT"],
221
+ ["Barkrowler", "seo", "Bark[rR]owler"],
222
+ ["TinEye", "seo", "TinEye"],
223
+ ["SocialRankIOBot", "seo", "SocialRankIOBot"],
224
+ ["trendictionbot", "seo", "trendictionbot"],
225
+ ["Ocarinabot", "seo", "Ocarinabot"],
226
+ ["epicbot", "seo", "epicbot"],
227
+ ["Primalbot", "seo", "Primalbot"],
228
+ ["DuckDuckGo-Favicons-Bot", "search", "DuckDuckGo-Favicons-Bot"],
229
+ ["GnowitNewsbot", "seo", "GnowitNewsbot"],
230
+ ["Leikibot", "seo", "Leikibot"],
231
+ ["YaK", "seo", "YaK\\/"],
232
+ ["PaperLiBot", "seo", "PaperLiBot"],
233
+ ["Digg Deeper", "seo", "Digg Deeper"],
234
+ ["dcrawl", "seo", "^dcrawl"],
235
+ ["Snacktory", "seo", "Snacktory"],
236
+ ["AndersPinkBot", "seo", "AndersPinkBot"],
237
+ ["Fyrebot", "seo", "Fyrebot"],
238
+ ["Luminator-robots", "seo", "Luminator-robots"],
239
+ ["ExtLinksBot", "seo", "ExtLinksBot"],
240
+ ["SurveyBot", "seo", "SurveyBot"],
241
+ ["Nuzzel", "seo", "Nuzzel"],
242
+ ["omgili", "seo", "omgili"],
243
+ ["PocketParser", "seo", "PocketParser"],
244
+ ["YisouSpider", "search", "YisouSpider"],
245
+ ["um-LN", "seo", "um-LN"],
246
+ ["ToutiaoSpider", "search", "ToutiaoSpider"],
247
+ ["MuckRack", "seo", "MuckRack"],
248
+ ["Jamie's Spider", "seo", "Jamie's Spider"],
249
+ ["Laserlikebot", "seo", "Laserlikebot"],
250
+ ["AppEngine-Google", "search", "AppEngine-Google"],
251
+ ["Upflow", "seo", "Upflow"],
252
+ ["Traackr.com", "seo", "Traackr\\.com"],
253
+ ["Twurly", "seo", "Twurly"],
254
+ ["DnyzBot", "seo", "DnyzBot"],
255
+ ["botify", "seo", "botify"],
256
+ ["007ac9 Crawler", "seo", "007ac9 Crawler"],
257
+ ["BehloolBot", "seo", "BehloolBot"],
258
+ ["BrandVerity", "seo", "BrandVerity"],
259
+ ["BDCbot", "seo", "BDCbot"],
260
+ ["ZumBot", "search", "ZumBot"],
261
+ ["EZID", "seo", "EZID"],
262
+ ["BLP_bbot", "seo", "BLP_bbot"],
263
+ ["BomboraBot", "seo", "BomboraBot"],
264
+ ["Companybook-Crawler", "seo", "Companybook-Crawler"],
265
+ ["Genieo", "seo", "Genieo"],
266
+ ["magpie-crawler", "seo", "magpie-crawler"],
267
+ ["MeltwaterNews", "seo", "MeltwaterNews"],
268
+ ["Moreover", "seo", "Moreover"],
269
+ ["ScoutJet", "seo", "ScoutJet"],
270
+ ["StorygizeBot", "seo", "StorygizeBot"],
271
+ ["seoscanners", "seo", "seoscanners"],
272
+ ["MauiBot", "seo", "MauiBot"],
273
+ ["AlphaBot", "seo", "AlphaBot"],
274
+ ["acapbot", "seo", "acapbot"],
275
+ ["bitlybot", "seo", "bitlybot"],
276
+ ["blogmuraBot", "seo", "blogmuraBot"],
277
+ ["Bot.AraTurka.com", "search", "Bot\\.AraTurka\\.com"],
278
+ ["bot-pge.chlooe.com", "seo", "bot-pge\\.chlooe\\.com"],
279
+ ["BoxcarBot", "seo", "BoxcarBot"],
280
+ ["Digincore bot", "seo", "Digincore bot"],
281
+ ["Disqus", "seo", "Disqus"],
282
+ ["Fetch", "seo", "Fetch\\/"],
283
+ ["Flamingo_SearchEngine", "search", "Flamingo_SearchEngine"],
284
+ ["G2 Web Services", "seo", "G2 Web Services"],
285
+ ["imrbot", "seo", "imrbot"],
286
+ ["Kemvibot", "seo", "Kemvibot"],
287
+ ["Landau-Media-Spider", "seo", "Landau-Media-Spider"],
288
+ ["linkapediabot", "seo", "linkapediabot"],
289
+ ["BLEXBot", "seo", "BLEXBot\\/"],
290
+ ["ZuperlistBot", "seo", "ZuperlistBot\\/"],
291
+ ["Diffbot", "seo", "Diffbot\\/"],
292
+ ["SEOkicks", "seo", "SEOkicks"],
293
+ ["tracemyfile", "seo", "tracemyfile"],
294
+ ["Nimbostratus-Bot", "seo", "Nimbostratus-Bot"],
295
+ ["PR-CY.RU", "seo", "PR-CY\\.RU"],
296
+ ["Datafeedwatch", "seo", "Datafeedwatch"],
297
+ ["TangibleeBot", "seo", "TangibleeBot"],
298
+ ["google-xrawler", "search", "google-xrawler"],
299
+ ["CloudFlare-Prefetch", "seo", "CloudFlare-Prefetch"],
300
+ ["Google-Structured-Data-Testing-Tool", "search", "Google-Structured-Data-Testing-Tool"],
301
+ ["WordupInfoSearch", "search", "WordupInfoSearch"],
302
+ ["WebDataStats", "seo", "WebDataStats"],
303
+ ["ZoomBot", "seo", "ZoomBot"],
304
+ ["VelenPublicWebCrawler", "seo", "VelenPublicWebCrawler"],
305
+ ["MoodleBot", "seo", "MoodleBot"],
306
+ ["jpg-newsbot", "seo", "jpg-newsbot"],
307
+ ["Google-PhysicalWeb", "search", "Google-PhysicalWeb"],
308
+ ["ICBot", "seo", "ICBot\\/"],
309
+ ["Twingly", "seo", "Twingly"],
310
+ ["Rivva", "seo", "Rivva"],
311
+ ["awesomecrawler", "seo", "awesomecrawler"],
312
+ ["Dataprovider.com", "seo", "Dataprovider\\.com"],
313
+ ["GroupHigh", "seo", "GroupHigh\\/"],
314
+ ["2ip.ru", "seo", "2ip\\.ru"],
315
+ ["Caliperbot", "seo", "Caliperbot"],
316
+ ["MBCrawler", "seo", "MBCrawler"],
317
+ ["online-webceo-bot", "seo", "online-webceo-bot"],
318
+ ["B2B Bot", "seo", "B2B Bot"],
319
+ ["AddSearchBot", "search", "AddSearchBot"],
320
+ ["Google Favicon", "search", "Google Favicon"],
321
+ ["HubSpot", "seo", "HubSpot"],
322
+ ["serpstatbot", "seo", "serpstatbot\\/"],
323
+ ["MixnodeCache", "seo", "MixnodeCache\\/"],
324
+ ["SimpleScraper", "seo", "SimpleScraper"],
325
+ ["Jooblebot", "search", "Jooblebot"],
326
+ ["RegionStuttgartBot", "seo", "RegionStuttgartBot"],
327
+ ["Datanyze", "seo", "Datanyze"],
328
+ ["Google-Site-Verification", "search", "Google-Site-Verification"],
329
+ ["TrendsmapResolver", "seo", "TrendsmapResolver"],
330
+ ["tweetedtimes", "seo", "tweetedtimes"],
331
+ ["NTENTbot", "search", "NTENTbot"],
332
+ ["SearchAtlas", "seo", "SearchAtlas"],
333
+ ["AmazonProductDiscovery", "search", "AmazonProductDiscovery"],
334
+ ["AmazonSellerInitiatedListing", "search", "AmazonSellerInitiatedListing"],
335
+ ["SerendeputyBot", "seo", "SerendeputyBot"],
336
+ ["officestorebot", "search", "officestorebot"],
337
+ ["Neticle Crawler", "seo", "Neticle Crawler"],
338
+ ["SurdotlyBot", "seo", "SurdotlyBot"],
339
+ ["LinkisBot", "seo", "LinkisBot"],
340
+ ["AwarioSmartBot", "seo", "AwarioSmartBot"],
341
+ ["AwarioRssBot", "seo", "AwarioRssBot"],
342
+ ["RyteBot", "seo", "RyteBot"],
343
+ ["AspiegelBot", "search", "AspiegelBot"],
344
+ ["zenback bot", "seo", "zenback bot"],
345
+ ["SentiBot", "seo", "SentiBot"],
346
+ ["Domains Project", "seo", "Domains Project\\/"],
347
+ ["Pandalytics", "seo", "Pandalytics"],
348
+ ["tigerbot", "seo", "tigerbot"],
349
+ ["PagePeeker", "seo", "PagePeeker\\/"],
350
+ ["startmebot", "seo", "startmebot\\/"],
351
+ ["JobboerseBot", "search", "JobboerseBot"],
352
+ ["seewithkids", "seo", "seewithkids"],
353
+ ["Cutbot", "seo", "Cutbot"],
354
+ ["BublupBot", "seo", "BublupBot"],
355
+ ["BrandONbot", "seo", "BrandONbot"],
356
+ ["RidderBot", "seo", "RidderBot"],
357
+ ["FindITAnswersbot", "seo", "FindITAnswersbot"],
358
+ ["infoobot", "seo", "infoobot"],
359
+ ["Refindbot", "seo", "Refindbot"],
360
+ ["SeobilityBot", "seo", "SeobilityBot"],
361
+ ["Cincraw", "seo", "Cincraw"],
362
+ ["Dragonbot", "seo", "Dragonbot"],
363
+ ["BitBot", "seo", "BitBot"],
364
+ ["Google-Certificates-Bridge", "search", "Google-Certificates-Bridge"],
365
+ ["centurybot", "seo", "centurybot"],
366
+ ["e.ventures Investment Crawler", "seo", "e\\.ventures Investment Crawler"],
367
+ ["evc-batch", "seo", "evc-batch"],
368
+ ["PetalBot", "search", "PetalBot"],
369
+ ["trovitBot", "search", "trovitBot"],
370
+ ["seostar.co", "seo", "seostar\\.co"],
371
+ ["IonCrawl", "seo", "IonCrawl"],
372
+ ["Seekport", "search", "Seekport"],
373
+ ["YandexRenderResourcesBot", "search", "YandexRenderResourcesBot\\/"],
374
+ ["ev-crawler", "seo", "ev-crawler\\/"],
375
+ ["woorankreview", "seo", "woorankreview\\/"],
376
+ ["Google-Safety", "search", "Google-Safety"],
377
+ ["AwarioBot", "seo", "AwarioBot"],
378
+ ["DataForSeoBot", "seo", "DataForSeoBot"],
379
+ ["WellKnownBot", "seo", "WellKnownBot"],
380
+ ["StractBot", "search", "StractBot"],
381
+ ["search.marginalia.nu", "search", "search\\.marginalia\\.nu"],
382
+ ["Nicecrawler", "search", "Nicecrawler"],
383
+ ["Neevabot", "search", "Neevabot"],
384
+ ["BrightEdge Crawler", "seo", "BrightEdge Crawler"],
385
+ ["SiteCheckerBotCrawler", "seo", "SiteCheckerBotCrawler"],
386
+ ["TombaPublicWebCrawler", "seo", "TombaPublicWebCrawler"],
387
+ ["CrawlyProjectCrawler", "seo", "CrawlyProjectCrawler"],
388
+ ["KomodiaBot", "seo", "KomodiaBot"],
389
+ ["KStandBot", "seo", "KStandBot"],
390
+ ["MTRobot", "seo", "MTRobot"],
391
+ ["hyscore.io", "seo", "hyscore\\.io"],
392
+ ["AlexandriaOrgBot", "search", "AlexandriaOrgBot"],
393
+ ["2ip bot", "seo", "2ip bot"],
394
+ ["Yellowbrandprotectionbot", "seo", "Yellowbrandprotectionbot"],
395
+ ["SEOlizer", "seo", "SEOlizer"],
396
+ ["vuhuvBot", "seo", "vuhuvBot"],
397
+ ["INETDEX-BOT", "search", "INETDEX-BOT"],
398
+ ["t3versionsBot", "seo", "t3versionsBot"],
399
+ ["Cocolyzebot", "seo", "Cocolyzebot"],
400
+ ["hypestat", "seo", "hypestat"],
401
+ ["sempi.tech", "seo", "sempi\\.tech"],
402
+ ["MetaInspector", "seo", "MetaInspector"],
403
+ ["OpenGraphCheck", "seo", "OpenGraphCheck"],
404
+ ["developers.google.com/+/web/snippet", "search", "developers\\.google\\.com\\/\\+\\/web\\/snippet"],
405
+ ["SenutoBot", "seo", "SenutoBot"],
406
+ ["PageThing", "seo", "PageThing"],
407
+ ["WordPress", "seo", "WordPress\\/"],
408
+ ["PhxBot", "search", "PhxBot"],
409
+ ["ImagesiftBot", "search", "ImagesiftBot"],
410
+ ["BW", "seo", "^BW\\/"],
411
+ ["GeedoBot", "search", "GeedoBot"],
412
+ ["Audisto Crawler", "seo", "Audisto Crawler"],
413
+ ["Vercelbot", "seo", "Vercelbot"],
414
+ ["vercel-screenshot", "seo", "vercel-screenshot"],
415
+ ["GeedoProductSearch", "search", "GeedoProductSearch"],
416
+ ["OnCrawl", "seo", "OnCrawl\\/"],
417
+ ["sitebulb", "seo", "sitebulb"],
418
+ ["YextBot", "seo", "YextBot\\/"],
419
+ ["Timpibot", "search", "Timpibot"],
420
+ ["SERankingBacklinksBot", "seo", "SERankingBacklinksBot"],
421
+ ["CMSChecker", "seo", "CMSChecker"],
422
+ ["OICrawler", "search", "OICrawler"],
423
+ ["SlaccaleBot", "seo", "SlaccaleBot"],
424
+ ["Gemini-Deep-Research", "search", "Gemini-Deep-Research"],
425
+ ["PerplexityUser", "search", "PerplexityUser"],
426
+ ["webzio", "seo", "webzio"],
427
+ ["Attracta", "seo", "Attracta"],
428
+ ["Cloudflare-Custom-Hostname-Verification", "seo", "Cloudflare-Custom-Hostname-Verification"],
429
+ ["Cloudflare-Stream-Webhook", "seo", "Cloudflare-Stream-Webhook"],
430
+ ["Cloudtrellis", "seo", "Cloudtrellis"],
431
+ ["Convermax", "seo", "Convermax"],
432
+ ["DeepCrawl", "seo", "DeepCrawl"],
433
+ ["FindFiles.net", "search", "FindFiles\\.net"],
434
+ ["FyndSearchEngine-Crawler", "search", "FyndSearchEngine-Crawler"],
435
+ ["FyndSearchEngine-ReCrawler", "search", "FyndSearchEngine-ReCrawler"],
436
+ ["Greppr Web Crawler", "search", "Greppr Web Crawler"],
437
+ ["IbouBot", "search", "IbouBot"],
438
+ ["LinkCheckerBot", "seo", "LinkCheckerBot"],
439
+ ["lyonl-asset-proxy", "search", "lyonl-asset-proxy"],
440
+ ["lyonl-crawler", "search", "lyonl-crawler"],
441
+ ["MagiBot", "search", "MagiBot"],
442
+ ["ModernizeBot", "seo", "ModernizeBot"],
443
+ ["NestDaddybot", "search", "NestDaddybot"],
444
+ ["Nitro", "seo", "Nitro-"],
445
+ ["NitroBot", "seo", "NitroBot"],
446
+ ["Qwantbot", "search", "Qwantbot"],
447
+ ["RSiteAuditor", "seo", "RSiteAuditor"],
448
+ ["Scrubby", "search", "Scrubby"],
449
+ ["seo4ajax.com", "seo", "seo4ajax\\.com"],
450
+ ["SMTnetPMBot", "search", "SMTnetPMBot"],
451
+ ["StatusNestBacklinkSpider", "seo", "StatusNestBacklinkSpider"],
452
+ ["stepstoneCrawlBot", "search", "stepstoneCrawlBot"],
453
+ ["WepchSearchEngine", "search", "WepchSearchEngine"],
454
+ ["XoviOnpageCrawler", "seo", "XoviOnpageCrawler"],
455
+ ["ZoomInfo", "search", "ZoomInfo-"],
456
+ ["7Siters", "search", "7Siters"],
457
+ ["CyberFindCrawler", "search", "CyberFindCrawler"],
458
+ ["FediIndex", "search", "FediIndex"],
459
+ ["fiperbot", "search", "fiperbot"],
460
+ ["FleebsBot", "search", "FleebsBot"],
461
+ ["MatchorySearch", "search", "MatchorySearch"],
462
+ ["NapBot", "search", "NapBot"],
463
+ ["seoLyt", "seo", "seoLyt"],
464
+ ["SitesOverPagesBot", "seo", "SitesOverPagesBot"],
465
+ ["Sosospider", "search", "Sosospider"],
466
+ ["WadooBot", "search", "WadooBot"],
467
+ ["Website-info.net-Robot", "seo", "Website-info\\.net-Robot"],
468
+ ["WikiDo", "search", "WikiDo"],
469
+ ["YoudaoBot", "search", "YoudaoBot"],
470
+ ["ZyBorg", "search", "ZyBorg"],
471
+ ["blekkobot", "search", "blekkobot"],
472
+ ["br-crawler", "search", "br-crawler"],
473
+ ["gsa-crawler", "search", "gsa-crawler"],
474
+ ["NaverBot", "search", "NaverBot"],
475
+ ["OpenTheBoxBot", "search", "OpenTheBoxBot"],
476
+ ["Swiftbot", "search", "Swiftbot"],
477
+ ["cognitiveseo.com", "seo", "cognitiveseo\\.com"],
478
+ ["XML Sitemaps Generator", "seo", "XML Sitemaps Generator"],
479
+ ["Zealbot", "search", "Zealbot"],
480
+ ["008", "seo", "008\\/"],
481
+ ["AdagioBot", "seo", "AdagioBot"],
482
+ ["AdminLabs", "seo", "AdminLabs"],
483
+ ["advanced_crawler", "seo", "advanced_crawler"],
484
+ ["Adventurer", "seo", "Adventurer"],
485
+ ["AGAKIDSBOT", "seo", "AGAKIDSBOT"],
486
+ ["AgencyAnalyticsBot", "seo", "AgencyAnalyticsBot"],
487
+ ["alexa site audit", "seo", "alexa site audit"],
488
+ ["Algolia Crawler", "search", "Algolia Crawler"],
489
+ ["alienfarm", "seo", "alienfarm"],
490
+ ["allOrigins", "seo", "allOrigins"],
491
+ ["KendraBot", "search", "KendraBot"],
492
+ ["AppSiteAssociation", "seo", "AppSiteAssociation"],
493
+ ["Aragog", "seo", "Aragog\\/"],
494
+ ["Asana", "seo", "Asana\\/"],
495
+ ["AudistoBot", "seo", "AudistoBot"],
496
+ ["BDBot", "seo", "BDBot\\/"],
497
+ ["BeeperBot", "seo", "BeeperBot"],
498
+ ["BigUpDataBot", "seo", "BigUpDataBot"],
499
+ ["Bitbucket-Webhooks", "seo", "Bitbucket-Webhooks"],
500
+ ["BoardGamePricesBot", "seo", "BoardGamePricesBot"],
501
+ ["BotPoke", "seo", "BotPoke"],
502
+ ["BDFetch", "seo", "BDFetch"],
503
+ ["Brandwatch", "seo", "Brandwatch"],
504
+ ["BraveBot", "search", "BraveBot"],
505
+ ["brokenlinkcheck.com", "seo", "brokenlinkcheck\\.com"],
506
+ ["BW", "seo", "BW\\/"],
507
+ ["Bushbaby", "seo", "Bushbaby"],
508
+ ["Butterfly", "seo", "Butterfly"],
509
+ ["CaliberBot", "seo", "CaliberBot"],
510
+ ["CapitalOneShopping", "seo", "CapitalOneShopping"],
511
+ ["centuryb.o.t9", "seo", "centuryb\\.o\\.t9"],
512
+ ["certytags", "seo", "certytags"],
513
+ ["ChargeBeeBot", "seo", "ChargeBeeBot"],
514
+ ["Charlotte", "seo", "Charlotte"],
515
+ ["Goodzer", "seo", "Goodzer"],
516
+ ["Chrome Privacy Preserving Prefetch Proxy", "seo", "Chrome Privacy Preserving Prefetch Proxy"],
517
+ ["CirrusExplorer", "seo", "CirrusExplorer"],
518
+ ["Clearscopebot", "seo", "Clearscopebot"],
519
+ ["WorldBot", "seo", "WorldBot"],
520
+ ["Cloudflare-Stream-Hook", "seo", "Cloudflare-Stream-Hook"],
521
+ ["cognitiveSEO Bot", "seo", "cognitiveSEO Bot"],
522
+ ["contentkingapp", "seo", "contentkingapp"],
523
+ ["Cotoyogi", "seo", "Cotoyogi"],
524
+ ["Coveobot", "search", "Coveobot"],
525
+ ["Crawlson", "seo", "Crawlson"],
526
+ ["RepoLookoutBot", "seo", "RepoLookoutBot"],
527
+ ["Criticalcss.com", "seo", "Criticalcss\\.com"],
528
+ ["DnBCrawler", "seo", "DnBCrawler"],
529
+ ["DomCopBot", "seo", "DomCopBot"],
530
+ ["Dlc", "seo", "Dlc\\/"],
531
+ ["easybill-ImportManager", "seo", "easybill-ImportManager"],
532
+ ["EchoboxBot", "seo", "EchoboxBot\\/"],
533
+ ["crusty", "seo", "crusty\\/"],
534
+ ["CXK_Bot", "seo", "CXK_Bot"],
535
+ ["daumoa", "search", "daumoa"],
536
+ ["DaspeedBot", "seo", "DaspeedBot"],
537
+ ["Dead Link Checker", "seo", "Dead Link Checker"],
538
+ ["Deskyobot", "seo", "Deskyobot"],
539
+ ["DF Bot", "seo", "DF Bot"],
540
+ ["Dmbot", "seo", "Dmbot"],
541
+ ["SustainabilityCrawler", "seo", "SustainabilityCrawler"],
542
+ ["edansbot", "seo", "edansbot"],
543
+ ["eMoneyBot", "seo", "eMoneyBot"],
544
+ ["EpivozCrawler", "seo", "EpivozCrawler"],
545
+ ["eRepublik.tools", "seo", "eRepublik\\.tools"],
546
+ ["ExodusMovement", "seo", "ExodusMovement"],
547
+ ["Ezgif", "seo", "Ezgif"],
548
+ ["factset_spyderbot", "seo", "factset_spyderbot"],
549
+ ["FDL Stats Bot", "seo", "FDL Stats Bot"],
550
+ ["Fedicabot", "seo", "Fedicabot"],
551
+ ["FedReporterDataBot", "seo", "FedReporterDataBot"],
552
+ ["fidget-spinner-bot", "seo", "fidget-spinner-bot"],
553
+ ["FirmoGraph", "seo", "FirmoGraph"],
554
+ ["Freespoke", "search", "Freespoke\\/"],
555
+ ["Friendly testing bot", "seo", "Friendly testing bot"],
556
+ ["friendly-spider", "seo", "friendly-spider"],
557
+ ["FriendlyCrawler", "seo", "FriendlyCrawler\\/"],
558
+ ["FullStoryBot", "seo", "FullStoryBot\\/"],
559
+ ["Funnelback", "search", "Funnelback"],
560
+ ["FuseonBot", "seo", "FuseonBot\\/"],
561
+ ["Gabanzabot", "search", "Gabanzabot\\/"],
562
+ ["gdnplus.com", "seo", "gdnplus\\.com"],
563
+ ["getthit.com", "seo", "getthit\\.com"],
564
+ ["GlobalWebSearch", "search", "GlobalWebSearch"],
565
+ ["Golfe", "search", "Golfe\\/"],
566
+ ["Google-Apps-Script", "seo", "Google-Apps-Script"],
567
+ ["GoogleAssociationService", "seo", "GoogleAssociationService\\/"],
568
+ ["GoogleProducer", "search", "GoogleProducer"],
569
+ ["Google-Area120", "seo", "Google-Area120"],
570
+ ["GoogleAssociationService", "seo", "GoogleAssociationService$"],
571
+ ["GoogleDocs", "seo", "GoogleDocs"],
572
+ ["GoPay", "seo", "GoPay"],
573
+ ["Grammarly", "seo", "Grammarly\\/"],
574
+ ["gregcrawler", "seo", "gregcrawler"],
575
+ ["Grover", "seo", "Grover\\/"],
576
+ ["GuestpostsBot", "seo", "GuestpostsBot\\/"],
577
+ ["HappyWing", "seo", "HappyWing"],
578
+ ["harsilbot", "seo", "harsilbot\\/"],
579
+ ["HawaiiBot", "seo", "HawaiiBot"],
580
+ ["hCardValidator", "seo", "hCardValidator"],
581
+ ["Hello World", "seo", "Hello World"],
582
+ ["HelloworkJobPostingBot", "seo", "HelloworkJobPostingBot\\/"],
583
+ ["HIFIBot", "seo", "HIFIBot\\/"],
584
+ ["Hlidam.to robot", "seo", "Hlidam\\.to robot"],
585
+ ["Hotjar", "seo", "Hotjar"],
586
+ ["Huckabot", "seo", "Huckabot\\/"],
587
+ ["Hype Machine", "seo", "Hype Machine\\/"],
588
+ ["Web Screen Service By hyperhost", "seo", "Web Screen Service By hyperhost"],
589
+ ["iAskBot", "search", "iAskBot\\/"],
590
+ ["IBM Crawler", "seo", "IBM Crawler"],
591
+ ["IFTTT", "seo", "IFTTT\\/"],
592
+ ["ImageFetcher", "seo", "ImageFetcher\\/"],
593
+ ["imgproxy", "seo", "imgproxy\\/"],
594
+ ["impendoom-bot", "seo", "impendoom-bot\\/"],
595
+ ["IndeedJobBot", "search", "IndeedJobBot"],
596
+ ["Innguma", "seo", "Innguma\\/"],
597
+ ["Instapaper", "seo", "Instapaper\\/"],
598
+ ["Integromat", "seo", "Integromat\\/"],
599
+ ["intelx.io_bot", "search", "intelx\\.io_bot"],
600
+ ["jobswithgptcom-bot", "search", "jobswithgptcom-bot"],
601
+ ["Jumio", "seo", "Jumio"],
602
+ ["Kagibot", "search", "Kagibot\\/"],
603
+ ["KangarooBot", "seo", "KangarooBot\\/"],
604
+ ["kazbtbot", "seo", "kazbtbot\\/"],
605
+ ["keys-so-bot", "seo", "keys-so-bot"],
606
+ ["kinsta-bot", "seo", "kinsta-bot"],
607
+ ["Klaviyo", "seo", "Klaviyo\\/"],
608
+ ["Kukei.eu-Bot", "seo", "Kukei\\.eu-Bot\\/"],
609
+ ["LastModBot", "seo", "LastModBot\\/"],
610
+ ["Level9SearchBot", "search", "Level9SearchBot\\/"],
611
+ ["LinkTiger", "seo", "LinkTiger"],
612
+ ["LinksIndexerBot", "seo", "LinksIndexerBot\\/"],
613
+ ["LinkWalker", "seo", "LinkWalker\\/"],
614
+ ["Macrobondbot", "seo", "Macrobondbot"],
615
+ ["MADBbot", "seo", "MADBbot\\/"],
616
+ ["Magellan", "seo", "Magellan"],
617
+ ["magicsearchdev", "seo", "magicsearchdev\\/"],
618
+ ["Magnet.me-web", "seo", "Magnet\\.me-web\\/"],
619
+ ["MainWP", "seo", "MainWP\\/"],
620
+ ["Make", "seo", "Make\\/"],
621
+ ["ManageWP", "seo", "ManageWP"],
622
+ ["MarketGoo", "seo", "MarketGoo\\/"],
623
+ ["MarketingMiner", "seo", "MarketingMiner"],
624
+ ["Mavifinds", "seo", "Mavifinds"],
625
+ ["MB-LinkChecker", "seo", "MB-LinkChecker"],
626
+ ["MedialogiaBot", "seo", "MedialogiaBot"],
627
+ ["MediaMonitoringBot", "seo", "MediaMonitoringBot\\/"],
628
+ ["Pywikibot", "seo", "Pywikibot\\/"],
629
+ ["CentComBot", "seo", "CentComBot\\/"],
630
+ ["MergadoBot", "seo", "MergadoBot"],
631
+ ["Metorik", "seo", "Metorik"],
632
+ ["MissinglettrBot", "seo", "MissinglettrBot\\/"],
633
+ ["crawler_eb_germany", "seo", "crawler_eb_germany"],
634
+ ["ModularConnector", "seo", "ModularConnector\\/"],
635
+ ["Mollie HTTP client", "seo", "Mollie HTTP client"],
636
+ ["Monibot", "seo", "Monibot"],
637
+ ["MonTools.com", "seo", "MonTools\\.com"],
638
+ ["MotoMinerBot", "seo", "MotoMinerBot\\/"],
639
+ ["my-tiny-bot", "seo", "my-tiny-bot"],
640
+ ["MyBot", "seo", "MyBot\\/"],
641
+ ["nbertaupete95", "seo", "nbertaupete95"],
642
+ ["NetAPI", "seo", "NetAPI"],
643
+ ["NetpeakCheckerBot", "seo", "NetpeakCheckerBot\\/"],
644
+ ["nyt_scraping", "seo", "nyt_scraping"],
645
+ ["NewsNow", "seo", "NewsNow\\/"],
646
+ ["nomore404.com robot", "seo", "nomore404\\.com robot"],
647
+ ["noorobot", "seo", "noorobot"],
648
+ ["Nooshub", "seo", "Nooshub\\/"],
649
+ ["Notabot", "seo", "Notabot"],
650
+ ["Novaact", "seo", "Novaact\\/"],
651
+ ["nvdorz", "seo", "nvdorz"],
652
+ ["Odin", "seo", "Odin;"],
653
+ ["Omnisend", "seo", "Omnisend\\/"],
654
+ ["Online Domain Tools", "seo", "Online Domain Tools"],
655
+ ["WebCEO Online", "seo", "WebCEO Online\\/"],
656
+ ["Owler (ows.eu", "search", "Owler \\(ows\\.eu"],
657
+ ["Operator", "seo", "Operator\\/"],
658
+ ["Orbbot", "seo", "Orbbot\\/"],
659
+ ["zebra-v2-bot", "seo", "zebra-v2-bot"],
660
+ ["OutsellURLValidator", "seo", "OutsellURLValidator"],
661
+ ["PanguBot", "search", "PanguBot"],
662
+ ["Paqlebot", "seo", "Paqlebot\\/"],
663
+ ["parse.ly scraper", "seo", "parse\\.ly scraper\\/"],
664
+ ["PayPal", "seo", "PayPal\\/"],
665
+ ["PDF24 URL To PDF", "seo", "PDF24 URL To PDF"],
666
+ ["PlayStore-Google", "search", "PlayStore-Google"],
667
+ ["Plesk screenshot bot", "seo", "Plesk screenshot bot"],
668
+ ["Potions", "seo", "Potions\\/"],
669
+ ["PressEngineBot", "seo", "PressEngineBot"],
670
+ ["PricedroneShoppingBot", "seo", "PricedroneShoppingBot\\/"],
671
+ ["PriEcoBot", "seo", "PriEcoBot\\/"],
672
+ ["PrintFriendly.com", "seo", "PrintFriendly\\.com"],
673
+ ["Pro-Sitemaps", "seo", "Pro-Sitemaps\\/"],
674
+ ["PS_Daily", "seo", "PS_Daily\\/"],
675
+ ["PWABuilderHttpAgent", "seo", "PWABuilderHttpAgent"],
676
+ ["QualifiedBot", "seo", "QualifiedBot\\/"],
677
+ ["rakutenusabot-image", "seo", "rakutenusabot-image\\/"],
678
+ ["top100.rambler.ru crawler", "search", "top100\\.rambler\\.ru crawler"],
679
+ ["RankurBot", "seo", "RankurBot\\/"],
680
+ ["RavenCrawler", "seo", "RavenCrawler\\/"],
681
+ ["Readable", "seo", "Readable\\/"],
682
+ ["Recurly Webhooks", "seo", "Recurly Webhooks\\/"],
683
+ ["Reelevant", "seo", "Reelevant\\/"],
684
+ ["remove.bg", "seo", "remove\\.bg\\/"],
685
+ ["Retool", "seo", "Retool\\/"],
686
+ ["RetroListeCOM", "seo", "RetroListeCOM\\/"],
687
+ ["RevvimGort", "seo", "RevvimGort\\/"],
688
+ ["reward-gateway", "seo", "reward-gateway"],
689
+ ["RobotsChecker", "seo", "RobotsChecker\\/"],
690
+ ["s4a-probe-bot", "seo", "s4a-probe-bot\\/"],
691
+ ["SFDC-Callout", "seo", "SFDC-Callout\\/"],
692
+ ["page-preview-tool", "seo", "page-preview-tool"],
693
+ ["SandobaCrawler", "seo", "SandobaCrawler\\/"],
694
+ ["GIFTEDVISITOR SCAN", "seo", "GIFTEDVISITOR SCAN"],
695
+ ["Schema-Markup-Validator", "seo", "Schema-Markup-Validator"],
696
+ ["Scoop.it", "seo", "Scoop\\.it\\/"],
697
+ ["ScrapeheroBot", "seo", "ScrapeheroBot\\/"],
698
+ ["screeenly-bot", "seo", "screeenly-bot"],
699
+ ["SEBot-WA", "seo", "SEBot-WA"],
700
+ ["Searcherweb", "search", "Searcherweb"],
701
+ ["Searcherxweb", "search", "Searcherxweb"],
702
+ ["SearchExpress", "search", "SearchExpress"],
703
+ ["semaltbot", "seo", "semaltbot\\/"],
704
+ ["SendGrid Event API", "seo", "SendGrid Event API"],
705
+ ["seo-audit-check-bot", "seo", "seo-audit-check-bot\\/"],
706
+ ["s4a", "seo", "s4a\\/"],
707
+ ["ClarityBot", "seo", "ClarityBot\\/"],
708
+ ["SeoSiteCheckup", "seo", "SeoSiteCheckup"],
709
+ ["SeoulBot", "seo", "SeoulBot"],
710
+ ["SERPtimizerBot", "seo", "SERPtimizerBot"],
711
+ ["ServerHunterSpider", "seo", "ServerHunterSpider\\/"],
712
+ ["SeznamHomepageCrawler", "search", "SeznamHomepageCrawler\\/"],
713
+ ["Shopify-Captain-Hook", "seo", "Shopify-Captain-Hook"],
714
+ ["Sidetrade indexer bot", "seo", "Sidetrade indexer bot"],
715
+ ["Silk", "seo", "^Silk\\/"],
716
+ ["SinceraSyntheticUser", "seo", "SinceraSyntheticUser\\/"],
717
+ ["Optimizer", "seo", "Optimizer\\)"],
718
+ ["SiteAuditBot", "seo", "SiteAuditBot\\/"],
719
+ ["SiteCheck-sitecrawl", "seo", "SiteCheck-sitecrawl"],
720
+ ["SiteScoreBot", "seo", "SiteScoreBot"],
721
+ ["SiteSearch360", "search", "SiteSearch360\\/"],
722
+ ["Konturbot", "seo", "Konturbot\\/"],
723
+ ["SkroutzBot", "seo", "SkroutzBot"],
724
+ ["SkyworkSpider", "seo", "SkyworkSpider"],
725
+ ["SlickBot", "seo", "SlickBot\\/"],
726
+ ["Snipcart", "seo", "Snipcart\\/"],
727
+ ["Sora POS", "seo", "Sora POS\\/"],
728
+ ["SparkShipping", "seo", "SparkShipping"],
729
+ ["SparkPost", "seo", "SparkPost"],
730
+ ["IDG/EU", "seo", "IDG\\/EU"],
731
+ ["SSSSBot", "seo", "SSSSBot\\/"],
732
+ ["Stape", "seo", "Stape\\/"],
733
+ ["StartpagePrivateImageProxy", "search", "StartpagePrivateImageProxy\\/"],
734
+ ["Statabot", "seo", "Statabot\\/"],
735
+ ["StatistikAustria", "seo", "StatistikAustria\\/"],
736
+ ["StatsDroneBot", "seo", "StatsDroneBot"],
737
+ ["Stripe", "seo", "Stripe\\/"],
738
+ ["Svix-Webhooks", "seo", "Svix-Webhooks\\/"],
739
+ ["SwifteqLinkChecker", "seo", "SwifteqLinkChecker"],
740
+ ["Swisscows", "search", "Swisscows"],
741
+ ["TactiScout", "seo", "TactiScout\\/"],
742
+ ["tchelebi", "seo", "tchelebi\\/"],
743
+ ["Testcrawler", "seo", "Testcrawler"],
744
+ ["test-bot", "seo", "test-bot"],
745
+ ["TestURI", "seo", "TestURI"],
746
+ ["TextRazor", "seo", "TextRazor"],
747
+ ["TheInternetSearchx", "search", "TheInternetSearchx"],
748
+ ["Trellis-Services", "seo", "Trellis-Services"],
749
+ ["trentwil.es", "seo", "trentwil\\.es"],
750
+ ["Trustly", "seo", "Trustly\\/"],
751
+ ["Tweakers", "seo", "Tweakers"],
752
+ ["UASlinkChecker", "seo", "UASlinkChecker\\/"],
753
+ ["Urlcheckr", "seo", "Urlcheckr\\/"],
754
+ ["URLSuMaBot", "seo", "URLSuMaBot"],
755
+ ["useeBookChecker", "seo", "useeBookChecker\\/"],
756
+ ["Vagabondo", "seo", "Vagabondo\\/"],
757
+ ["videootvBot", "seo", "videootvBot"],
758
+ ["VsuSearchSpider", "search", "VsuSearchSpider\\/"],
759
+ ["WARDBot", "seo", "WARDBot\\/"],
760
+ ["Webflow", "seo", "Webflow"],
761
+ ["webprosbot", "seo", "webprosbot\\/"],
762
+ ["WebSniffer", "seo", "WebSniffer\\/"],
763
+ ["WSM", "seo", "WSM\\/"],
764
+ ["WebwikiBot", "seo", "WebwikiBot\\/"],
765
+ ["WhatsMyIP.org", "seo", "WhatsMyIP\\.org"],
766
+ ["Wheregoes.com", "seo", "Wheregoes\\.com"],
767
+ ["WireReaderBot", "seo", "WireReaderBot\\/"],
768
+ ["wmtips.com", "seo", "wmtips\\.com\\/"],
769
+ ["WordCountBot", "seo", "WordCountBot\\/"],
770
+ ["Wordup-1", "seo", "Wordup-1"],
771
+ ["workona-favicon-service", "seo", "workona-favicon-service\\/"],
772
+ ["Indy Library", "seo", "Indy Library"],
773
+ ["WJHRO", "seo", "WJHRO\\/"],
774
+ ["WovnCrawler", "seo", "WovnCrawler\\/"],
775
+ ["wowLink Crawler", "seo", "wowLink Crawler\\/"],
776
+ ["WP Time Capsule", "seo", "WP Time Capsule"],
777
+ ["wpbot", "seo", "wpbot\\/"],
778
+ ["WPMU DEV Broken Link Checker", "seo", "WPMU DEV Broken Link Checker"],
779
+ ["WRTNBot", "seo", "WRTNBot"],
780
+ ["YahooCacheSystem", "seo", "YahooCacheSystem"],
781
+ ["YokoyGroupAG", "seo", "YokoyGroupAG\\/"],
782
+ ["Yuuperbot", "seo", "Yuuperbot"],
783
+ ["Zapier", "seo", "Zapier"],
784
+ ["Zendesk Webhook", "seo", "Zendesk Webhook"],
785
+ ["Zombiebot", "seo", "Zombiebot\\/"],
786
+ ["zzhbot", "seo", "zzhbot"],
787
+ ["Penthouse Critical Path CSS Generator", "seo", "Penthouse Critical Path CSS Generator"],
788
+ ["Skroutz ImageBot", "seo", "Skroutz ImageBot"],
789
+ ["GeedoShopProductFinder", "search", "GeedoShopProductFinder"],
790
+ ["IsonSearchBot", "search", "IsonSearchBot"],
791
+ ];