legitbot 1.12.12 → 1.13.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/autocorrect.yml +9 -6
- data/.github/workflows/pr.yml +11 -5
- data/.github/workflows/release.yml +5 -4
- data/.github/workflows/scan-gha.yml +20 -0
- data/.rubocop.yml +2 -1
- data/.ruby-version +1 -1
- data/README.md +1 -0
- data/legitbot.gemspec +1 -1
- data/lib/legitbot/duckduckgo.rb +90 -0
- data/lib/legitbot/openai.rb +16 -3
- data/lib/legitbot/perplexity.rb +34 -0
- data/lib/legitbot/version.rb +1 -1
- data/lib/legitbot.rb +1 -0
- metadata +6 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: de4a0b7762818f68d6ce586c9fd8a8b21a4cc0c7134e1ab9c24f85608509d49b
|
|
4
|
+
data.tar.gz: 331c4af109e383fbfeb28351ab34bb19df3522e9e6a79ee881e0fd4420bfeeb9
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9c68b237016fae690bdb18d61c31cd4a99bf68b932b89d8d7158b6b6412c6e32eb64ca5812e80fe8805c4afb254bf5929cff6b7a4609398b486e7b4c87287892
|
|
7
|
+
data.tar.gz: 3280e852b8efc9121e3e875b8d050f09ffc086e27d1ddd948d9d7311fb7fe69ebf4fdcb0a7d90f105fc9326c97cdbe022237f3925bf35da6f9b76ec7ffd570e2
|
|
@@ -9,20 +9,23 @@ on:
|
|
|
9
9
|
jobs:
|
|
10
10
|
lint:
|
|
11
11
|
runs-on: ubuntu-latest
|
|
12
|
+
permissions:
|
|
13
|
+
contents: write
|
|
14
|
+
pull-requests: write
|
|
12
15
|
|
|
13
16
|
steps:
|
|
14
|
-
- uses: actions/checkout@
|
|
17
|
+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
18
|
+
with:
|
|
19
|
+
persist-credentials: false
|
|
15
20
|
- name: Set up Ruby
|
|
16
|
-
uses: ruby/setup-ruby@v1
|
|
21
|
+
uses: ruby/setup-ruby@6aaa311d81eba98ae12eaffbcb63296ace0efcde # v1.307.0
|
|
17
22
|
with:
|
|
18
23
|
bundler-cache: true
|
|
19
|
-
- name: Run tests
|
|
20
|
-
run: bundle exec rake test
|
|
21
24
|
- name: Run linter
|
|
22
|
-
run: bundle exec rubocop -
|
|
25
|
+
run: bundle exec rubocop --only Custom/IpRanges -x
|
|
23
26
|
continue-on-error: true
|
|
24
27
|
- name: Create Pull Request
|
|
25
|
-
uses: peter-evans/create-pull-request@
|
|
28
|
+
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
|
26
29
|
with:
|
|
27
30
|
branch: update/lint-autocorrect
|
|
28
31
|
delete-branch: true
|
data/.github/workflows/pr.yml
CHANGED
|
@@ -7,16 +7,19 @@ jobs:
|
|
|
7
7
|
test:
|
|
8
8
|
name: Tests
|
|
9
9
|
runs-on: ubuntu-latest
|
|
10
|
+
permissions: {}
|
|
10
11
|
|
|
11
12
|
strategy:
|
|
12
13
|
fail-fast: false
|
|
13
14
|
matrix:
|
|
14
|
-
ruby: [jruby, 3.
|
|
15
|
+
ruby: [jruby, 3.2, 3.3, 3.4, 4.0]
|
|
15
16
|
|
|
16
17
|
steps:
|
|
17
|
-
- uses: actions/checkout@
|
|
18
|
+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
19
|
+
with:
|
|
20
|
+
persist-credentials: false
|
|
18
21
|
- name: Set up Ruby
|
|
19
|
-
uses: ruby/setup-ruby@v1
|
|
22
|
+
uses: ruby/setup-ruby@6aaa311d81eba98ae12eaffbcb63296ace0efcde # v1.307.0
|
|
20
23
|
with:
|
|
21
24
|
ruby-version: ${{ matrix.ruby }}
|
|
22
25
|
bundler-cache: true
|
|
@@ -27,11 +30,14 @@ jobs:
|
|
|
27
30
|
name: Lint
|
|
28
31
|
needs: test
|
|
29
32
|
runs-on: ubuntu-latest
|
|
33
|
+
permissions: {}
|
|
30
34
|
|
|
31
35
|
steps:
|
|
32
|
-
- uses: actions/checkout@
|
|
36
|
+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
37
|
+
with:
|
|
38
|
+
persist-credentials: false
|
|
33
39
|
- name: Set up Ruby
|
|
34
|
-
uses: ruby/setup-ruby@v1
|
|
40
|
+
uses: ruby/setup-ruby@6aaa311d81eba98ae12eaffbcb63296ace0efcde # v1.307.0
|
|
35
41
|
with:
|
|
36
42
|
bundler-cache: true
|
|
37
43
|
- name: Run linter
|
|
@@ -17,14 +17,15 @@ jobs:
|
|
|
17
17
|
contents: write
|
|
18
18
|
|
|
19
19
|
steps:
|
|
20
|
-
- uses: actions/checkout@
|
|
20
|
+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
21
21
|
with:
|
|
22
22
|
persist-credentials: false
|
|
23
23
|
- name: Set up Ruby
|
|
24
|
-
uses: ruby/setup-ruby@v1
|
|
24
|
+
uses: ruby/setup-ruby@6aaa311d81eba98ae12eaffbcb63296ace0efcde # v1.307.0
|
|
25
25
|
with:
|
|
26
|
-
bundler-cache:
|
|
26
|
+
bundler-cache: false
|
|
27
|
+
- run: bundle install
|
|
27
28
|
- name: Run tests
|
|
28
29
|
run: bundle exec rake test
|
|
29
30
|
- name: Publish
|
|
30
|
-
uses: rubygems/release-gem@v1
|
|
31
|
+
uses: rubygems/release-gem@6317d8d1f7e28c24d28f6eff169ea854948bd9f7 # v1.2.0
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
name: GitHub Actions
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- master
|
|
7
|
+
pull_request:
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
gha-scan:
|
|
11
|
+
name: Security Scan
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
permissions:
|
|
14
|
+
security-events: write
|
|
15
|
+
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
18
|
+
with:
|
|
19
|
+
persist-credentials: false
|
|
20
|
+
- uses: zizmorcore/zizmor-action@b572f7b1a1c2d41efaab43d504f68d215c3cd727 # v0.5.4
|
data/.rubocop.yml
CHANGED
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
3.
|
|
1
|
+
3.3.11
|
data/README.md
CHANGED
|
@@ -65,6 +65,7 @@ end
|
|
|
65
65
|
- [Oracle Data Cloud Crawler](https://www.oracle.com/corporate/acquisitions/grapeshot/crawler.html)
|
|
66
66
|
- [Marginalia](https://www.marginalia.nu/marginalia-search/for-webmasters/)
|
|
67
67
|
- [Meta / Facebook Web crawlers](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/)
|
|
68
|
+
- [Perplexity](https://docs.perplexity.ai/docs/resources/perplexity-crawlers#ip-address-sources)
|
|
68
69
|
- [Petal search engine](http://aspiegel.com/petalbot)
|
|
69
70
|
- [Pinterest](https://help.pinterest.com/en/articles/about-pinterest-crawler-0)
|
|
70
71
|
- [Twitterbot](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/getting-started),
|
data/legitbot.gemspec
CHANGED
|
@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
|
|
|
19
19
|
'rubygems_mfa_required' => 'true'
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
-
spec.required_ruby_version = '>= 3.
|
|
22
|
+
spec.required_ruby_version = '>= 3.2.0'
|
|
23
23
|
spec.add_dependency 'fast_interval_tree', '~> 0.2', '>= 0.2.2'
|
|
24
24
|
spec.add_dependency 'irrc', '~> 0.2', '>= 0.2.1'
|
|
25
25
|
spec.add_dependency 'logger', '~> 1.7', '>= 1.7.0'
|
data/lib/legitbot/duckduckgo.rb
CHANGED
|
@@ -9,8 +9,12 @@ module Legitbot # :nodoc:
|
|
|
9
9
|
ip_ranges %w[
|
|
10
10
|
4.144.182.50
|
|
11
11
|
4.150.142.218
|
|
12
|
+
4.156.30.123
|
|
12
13
|
4.156.136.79
|
|
13
14
|
4.156.154.107
|
|
15
|
+
4.172.49.103
|
|
16
|
+
4.177.16.196
|
|
17
|
+
4.182.10.198
|
|
14
18
|
4.182.131.108
|
|
15
19
|
4.195.133.120
|
|
16
20
|
4.201.125.59
|
|
@@ -21,20 +25,27 @@ module Legitbot # :nodoc:
|
|
|
21
25
|
4.207.220.92
|
|
22
26
|
4.209.224.56
|
|
23
27
|
4.213.46.14
|
|
28
|
+
4.226.40.135
|
|
24
29
|
4.228.76.163
|
|
30
|
+
4.237.244.80
|
|
31
|
+
4.248.234.26
|
|
25
32
|
4.249.216.104
|
|
33
|
+
4.254.123.56
|
|
26
34
|
4.255.35.121
|
|
35
|
+
13.86.35.212
|
|
27
36
|
13.89.106.77
|
|
28
37
|
20.3.1.178
|
|
29
38
|
20.8.252.26
|
|
30
39
|
20.12.141.99
|
|
31
40
|
20.13.44.19
|
|
32
41
|
20.40.133.240
|
|
42
|
+
20.40.147.172
|
|
33
43
|
20.43.150.85
|
|
34
44
|
20.43.150.93
|
|
35
45
|
20.43.152.28
|
|
36
46
|
20.43.172.120
|
|
37
47
|
20.44.222.1
|
|
48
|
+
20.49.129.236
|
|
38
49
|
20.49.136.28
|
|
39
50
|
20.50.48.159
|
|
40
51
|
20.50.48.192
|
|
@@ -64,6 +75,7 @@ module Legitbot # :nodoc:
|
|
|
64
75
|
20.54.224.39
|
|
65
76
|
20.56.197.58
|
|
66
77
|
20.56.197.63
|
|
78
|
+
20.59.0.65
|
|
67
79
|
20.61.34.40
|
|
68
80
|
20.61.142.192
|
|
69
81
|
20.62.224.44
|
|
@@ -71,21 +83,33 @@ module Legitbot # :nodoc:
|
|
|
71
83
|
20.69.129.205
|
|
72
84
|
20.69.131.45
|
|
73
85
|
20.71.12.143
|
|
86
|
+
20.71.69.210
|
|
87
|
+
20.72.203.250
|
|
88
|
+
20.72.221.45
|
|
74
89
|
20.72.242.93
|
|
75
90
|
20.73.132.240
|
|
76
91
|
20.73.202.147
|
|
77
92
|
20.75.144.152
|
|
93
|
+
20.77.146.108
|
|
78
94
|
20.79.226.26
|
|
79
95
|
20.79.238.198
|
|
80
96
|
20.79.239.66
|
|
81
97
|
20.80.129.80
|
|
98
|
+
20.80.135.7
|
|
82
99
|
20.80.156.88
|
|
83
100
|
20.93.28.24
|
|
101
|
+
20.99.154.113
|
|
84
102
|
20.99.255.235
|
|
103
|
+
20.100.136.36
|
|
104
|
+
20.100.140.155
|
|
105
|
+
20.101.17.173
|
|
85
106
|
20.112.58.44
|
|
86
107
|
20.113.3.121
|
|
87
108
|
20.113.14.159
|
|
109
|
+
20.118.11.251
|
|
88
110
|
20.166.171.150
|
|
111
|
+
20.170.75.54
|
|
112
|
+
20.175.232.228
|
|
89
113
|
20.185.79.15
|
|
90
114
|
20.185.79.47
|
|
91
115
|
20.191.44.16
|
|
@@ -119,13 +143,17 @@ module Legitbot # :nodoc:
|
|
|
119
143
|
20.207.99.197
|
|
120
144
|
20.207.107.181
|
|
121
145
|
20.212.90.107
|
|
146
|
+
20.216.200.223
|
|
122
147
|
20.219.43.246
|
|
123
148
|
20.219.45.67
|
|
124
149
|
20.219.45.190
|
|
125
150
|
20.226.133.105
|
|
126
151
|
20.232.51.46
|
|
152
|
+
20.242.135.165
|
|
153
|
+
20.250.51.113
|
|
127
154
|
20.252.16.193
|
|
128
155
|
20.253.59.76
|
|
156
|
+
20.253.96.199
|
|
129
157
|
40.64.105.247
|
|
130
158
|
40.64.106.11
|
|
131
159
|
40.76.162.191
|
|
@@ -135,6 +163,7 @@ module Legitbot # :nodoc:
|
|
|
135
163
|
40.76.163.23
|
|
136
164
|
40.76.173.151
|
|
137
165
|
40.81.250.205
|
|
166
|
+
40.82.218.203
|
|
138
167
|
40.88.21.235
|
|
139
168
|
40.89.243.175
|
|
140
169
|
40.114.182.45
|
|
@@ -148,7 +177,9 @@ module Legitbot # :nodoc:
|
|
|
148
177
|
40.119.232.215
|
|
149
178
|
40.119.232.218
|
|
150
179
|
40.119.232.251
|
|
180
|
+
40.121.121.54
|
|
151
181
|
40.127.154.196
|
|
182
|
+
48.200.33.227
|
|
152
183
|
48.204.32.87
|
|
153
184
|
48.217.23.236
|
|
154
185
|
48.217.129.210
|
|
@@ -196,6 +227,7 @@ module Legitbot # :nodoc:
|
|
|
196
227
|
52.143.243.117
|
|
197
228
|
52.143.244.81
|
|
198
229
|
52.143.247.235
|
|
230
|
+
52.146.57.51
|
|
199
231
|
52.146.58.236
|
|
200
232
|
52.146.59.12
|
|
201
233
|
52.146.59.154
|
|
@@ -237,6 +269,7 @@ module Legitbot # :nodoc:
|
|
|
237
269
|
52.154.171.235
|
|
238
270
|
52.154.171.250
|
|
239
271
|
52.154.172.2
|
|
272
|
+
52.154.204.93
|
|
240
273
|
52.186.37.211
|
|
241
274
|
52.188.89.106
|
|
242
275
|
52.190.37.160
|
|
@@ -263,13 +296,17 @@ module Legitbot # :nodoc:
|
|
|
263
296
|
52.224.21.53
|
|
264
297
|
52.224.21.55
|
|
265
298
|
52.224.21.61
|
|
299
|
+
52.224.191.245
|
|
266
300
|
52.242.224.168
|
|
267
301
|
52.250.46.221
|
|
268
302
|
57.152.72.128
|
|
269
303
|
57.155.18.44
|
|
304
|
+
57.167.88.38
|
|
305
|
+
64.236.93.100
|
|
270
306
|
64.236.118.43
|
|
271
307
|
68.219.152.220
|
|
272
308
|
74.179.232.116
|
|
309
|
+
74.242.194.48
|
|
273
310
|
104.43.54.127
|
|
274
311
|
104.43.55.116
|
|
275
312
|
104.43.55.117
|
|
@@ -277,9 +314,15 @@ module Legitbot # :nodoc:
|
|
|
277
314
|
104.43.55.167
|
|
278
315
|
108.141.83.74
|
|
279
316
|
128.203.132.152
|
|
317
|
+
128.203.214.241
|
|
280
318
|
130.107.228.224
|
|
319
|
+
131.145.39.118
|
|
320
|
+
131.189.183.241
|
|
281
321
|
132.164.209.198
|
|
322
|
+
132.220.1.71
|
|
323
|
+
134.33.207.189
|
|
282
324
|
135.234.221.112
|
|
325
|
+
172.168.43.43
|
|
283
326
|
172.168.53.53
|
|
284
327
|
172.168.115.250
|
|
285
328
|
172.168.227.120
|
|
@@ -288,6 +331,8 @@ module Legitbot # :nodoc:
|
|
|
288
331
|
172.169.28.184
|
|
289
332
|
172.169.60.134
|
|
290
333
|
172.169.177.131
|
|
334
|
+
172.189.40.18
|
|
335
|
+
172.193.197.146
|
|
291
336
|
172.193.245.229
|
|
292
337
|
172.194.141.49
|
|
293
338
|
172.199.55.212
|
|
@@ -310,8 +355,12 @@ module Legitbot # :nodoc:
|
|
|
310
355
|
ip_ranges %w[
|
|
311
356
|
4.144.182.50
|
|
312
357
|
4.150.142.218
|
|
358
|
+
4.156.30.123
|
|
313
359
|
4.156.136.79
|
|
314
360
|
4.156.154.107
|
|
361
|
+
4.172.49.103
|
|
362
|
+
4.177.16.196
|
|
363
|
+
4.182.10.198
|
|
315
364
|
4.182.131.108
|
|
316
365
|
4.195.133.120
|
|
317
366
|
4.201.125.59
|
|
@@ -322,20 +371,27 @@ module Legitbot # :nodoc:
|
|
|
322
371
|
4.207.220.92
|
|
323
372
|
4.209.224.56
|
|
324
373
|
4.213.46.14
|
|
374
|
+
4.226.40.135
|
|
325
375
|
4.228.76.163
|
|
376
|
+
4.237.244.80
|
|
377
|
+
4.248.234.26
|
|
326
378
|
4.249.216.104
|
|
379
|
+
4.254.123.56
|
|
327
380
|
4.255.35.121
|
|
381
|
+
13.86.35.212
|
|
328
382
|
13.89.106.77
|
|
329
383
|
20.3.1.178
|
|
330
384
|
20.8.252.26
|
|
331
385
|
20.12.141.99
|
|
332
386
|
20.13.44.19
|
|
333
387
|
20.40.133.240
|
|
388
|
+
20.40.147.172
|
|
334
389
|
20.43.150.85
|
|
335
390
|
20.43.150.93
|
|
336
391
|
20.43.152.28
|
|
337
392
|
20.43.172.120
|
|
338
393
|
20.44.222.1
|
|
394
|
+
20.49.129.236
|
|
339
395
|
20.49.136.28
|
|
340
396
|
20.50.48.159
|
|
341
397
|
20.50.48.192
|
|
@@ -365,6 +421,7 @@ module Legitbot # :nodoc:
|
|
|
365
421
|
20.54.224.39
|
|
366
422
|
20.56.197.58
|
|
367
423
|
20.56.197.63
|
|
424
|
+
20.59.0.65
|
|
368
425
|
20.61.34.40
|
|
369
426
|
20.61.142.192
|
|
370
427
|
20.62.224.44
|
|
@@ -372,21 +429,33 @@ module Legitbot # :nodoc:
|
|
|
372
429
|
20.69.129.205
|
|
373
430
|
20.69.131.45
|
|
374
431
|
20.71.12.143
|
|
432
|
+
20.71.69.210
|
|
433
|
+
20.72.203.250
|
|
434
|
+
20.72.221.45
|
|
375
435
|
20.72.242.93
|
|
376
436
|
20.73.132.240
|
|
377
437
|
20.73.202.147
|
|
378
438
|
20.75.144.152
|
|
439
|
+
20.77.146.108
|
|
379
440
|
20.79.226.26
|
|
380
441
|
20.79.238.198
|
|
381
442
|
20.79.239.66
|
|
382
443
|
20.80.129.80
|
|
444
|
+
20.80.135.7
|
|
383
445
|
20.80.156.88
|
|
384
446
|
20.93.28.24
|
|
447
|
+
20.99.154.113
|
|
385
448
|
20.99.255.235
|
|
449
|
+
20.100.136.36
|
|
450
|
+
20.100.140.155
|
|
451
|
+
20.101.17.173
|
|
386
452
|
20.112.58.44
|
|
387
453
|
20.113.3.121
|
|
388
454
|
20.113.14.159
|
|
455
|
+
20.118.11.251
|
|
389
456
|
20.166.171.150
|
|
457
|
+
20.170.75.54
|
|
458
|
+
20.175.232.228
|
|
390
459
|
20.185.79.15
|
|
391
460
|
20.185.79.47
|
|
392
461
|
20.191.44.16
|
|
@@ -420,13 +489,17 @@ module Legitbot # :nodoc:
|
|
|
420
489
|
20.207.99.197
|
|
421
490
|
20.207.107.181
|
|
422
491
|
20.212.90.107
|
|
492
|
+
20.216.200.223
|
|
423
493
|
20.219.43.246
|
|
424
494
|
20.219.45.67
|
|
425
495
|
20.219.45.190
|
|
426
496
|
20.226.133.105
|
|
427
497
|
20.232.51.46
|
|
498
|
+
20.242.135.165
|
|
499
|
+
20.250.51.113
|
|
428
500
|
20.252.16.193
|
|
429
501
|
20.253.59.76
|
|
502
|
+
20.253.96.199
|
|
430
503
|
40.64.105.247
|
|
431
504
|
40.64.106.11
|
|
432
505
|
40.76.162.191
|
|
@@ -436,6 +509,7 @@ module Legitbot # :nodoc:
|
|
|
436
509
|
40.76.163.23
|
|
437
510
|
40.76.173.151
|
|
438
511
|
40.81.250.205
|
|
512
|
+
40.82.218.203
|
|
439
513
|
40.88.21.235
|
|
440
514
|
40.89.243.175
|
|
441
515
|
40.114.182.45
|
|
@@ -449,7 +523,9 @@ module Legitbot # :nodoc:
|
|
|
449
523
|
40.119.232.215
|
|
450
524
|
40.119.232.218
|
|
451
525
|
40.119.232.251
|
|
526
|
+
40.121.121.54
|
|
452
527
|
40.127.154.196
|
|
528
|
+
48.200.33.227
|
|
453
529
|
48.204.32.87
|
|
454
530
|
48.217.23.236
|
|
455
531
|
48.217.129.210
|
|
@@ -497,6 +573,7 @@ module Legitbot # :nodoc:
|
|
|
497
573
|
52.143.243.117
|
|
498
574
|
52.143.244.81
|
|
499
575
|
52.143.247.235
|
|
576
|
+
52.146.57.51
|
|
500
577
|
52.146.58.236
|
|
501
578
|
52.146.59.12
|
|
502
579
|
52.146.59.154
|
|
@@ -538,6 +615,7 @@ module Legitbot # :nodoc:
|
|
|
538
615
|
52.154.171.235
|
|
539
616
|
52.154.171.250
|
|
540
617
|
52.154.172.2
|
|
618
|
+
52.154.204.93
|
|
541
619
|
52.186.37.211
|
|
542
620
|
52.188.89.106
|
|
543
621
|
52.190.37.160
|
|
@@ -564,13 +642,17 @@ module Legitbot # :nodoc:
|
|
|
564
642
|
52.224.21.53
|
|
565
643
|
52.224.21.55
|
|
566
644
|
52.224.21.61
|
|
645
|
+
52.224.191.245
|
|
567
646
|
52.242.224.168
|
|
568
647
|
52.250.46.221
|
|
569
648
|
57.152.72.128
|
|
570
649
|
57.155.18.44
|
|
650
|
+
57.167.88.38
|
|
651
|
+
64.236.93.100
|
|
571
652
|
64.236.118.43
|
|
572
653
|
68.219.152.220
|
|
573
654
|
74.179.232.116
|
|
655
|
+
74.242.194.48
|
|
574
656
|
104.43.54.127
|
|
575
657
|
104.43.55.116
|
|
576
658
|
104.43.55.117
|
|
@@ -578,9 +660,15 @@ module Legitbot # :nodoc:
|
|
|
578
660
|
104.43.55.167
|
|
579
661
|
108.141.83.74
|
|
580
662
|
128.203.132.152
|
|
663
|
+
128.203.214.241
|
|
581
664
|
130.107.228.224
|
|
665
|
+
131.145.39.118
|
|
666
|
+
131.189.183.241
|
|
582
667
|
132.164.209.198
|
|
668
|
+
132.220.1.71
|
|
669
|
+
134.33.207.189
|
|
583
670
|
135.234.221.112
|
|
671
|
+
172.168.43.43
|
|
584
672
|
172.168.53.53
|
|
585
673
|
172.168.115.250
|
|
586
674
|
172.168.227.120
|
|
@@ -589,6 +677,8 @@ module Legitbot # :nodoc:
|
|
|
589
677
|
172.169.28.184
|
|
590
678
|
172.169.60.134
|
|
591
679
|
172.169.177.131
|
|
680
|
+
172.189.40.18
|
|
681
|
+
172.193.197.146
|
|
592
682
|
172.193.245.229
|
|
593
683
|
172.194.141.49
|
|
594
684
|
172.199.55.212
|
data/lib/legitbot/openai.rb
CHANGED
|
@@ -52,8 +52,10 @@ module Legitbot # :nodoc:
|
|
|
52
52
|
4.197.115.112/28
|
|
53
53
|
4.198.72.16/28
|
|
54
54
|
4.205.128.176/28
|
|
55
|
+
4.226.226.32/28
|
|
55
56
|
9.160.163.224/28
|
|
56
57
|
9.160.164.128/28
|
|
58
|
+
9.234.96.192/28
|
|
57
59
|
13.65.138.96/28
|
|
58
60
|
13.65.138.112/28
|
|
59
61
|
13.67.72.16/28
|
|
@@ -71,9 +73,12 @@ module Legitbot # :nodoc:
|
|
|
71
73
|
20.42.250.32/28
|
|
72
74
|
20.45.178.144/28
|
|
73
75
|
20.55.229.144/28
|
|
76
|
+
20.57.199.192/28
|
|
74
77
|
20.63.221.64/28
|
|
75
78
|
20.97.189.96/28
|
|
76
79
|
20.102.212.144/28
|
|
80
|
+
20.113.218.16/28
|
|
81
|
+
20.113.225.112/28
|
|
77
82
|
20.125.112.224/28
|
|
78
83
|
20.125.144.144/28
|
|
79
84
|
20.161.75.208/28
|
|
@@ -105,6 +110,7 @@ module Legitbot # :nodoc:
|
|
|
105
110
|
20.194.1.0/28
|
|
106
111
|
20.194.157.176/28
|
|
107
112
|
20.198.67.96/28
|
|
113
|
+
20.199.211.160/28
|
|
108
114
|
20.203.245.32/28
|
|
109
115
|
20.204.24.240/28
|
|
110
116
|
20.206.107.192/28
|
|
@@ -153,11 +159,16 @@ module Legitbot # :nodoc:
|
|
|
153
159
|
40.84.221.224/28
|
|
154
160
|
40.116.73.208/28
|
|
155
161
|
40.122.235.112/28
|
|
162
|
+
48.193.44.32/28
|
|
156
163
|
51.8.155.48/28
|
|
157
164
|
51.8.155.64/28
|
|
158
165
|
51.8.155.80/28
|
|
159
166
|
51.107.70.192/28
|
|
167
|
+
51.116.2.64/28
|
|
168
|
+
51.116.2.80/28
|
|
160
169
|
52.148.129.32/28
|
|
170
|
+
52.153.130.48/28
|
|
171
|
+
52.153.130.64/28
|
|
161
172
|
52.154.22.48/28
|
|
162
173
|
52.156.77.144/28
|
|
163
174
|
52.159.227.32/28
|
|
@@ -193,6 +204,7 @@ module Legitbot # :nodoc:
|
|
|
193
204
|
52.231.49.48/28
|
|
194
205
|
52.231.50.64/28
|
|
195
206
|
52.236.94.144/28
|
|
207
|
+
52.241.146.208/28
|
|
196
208
|
52.242.132.224/28
|
|
197
209
|
52.242.132.240/28
|
|
198
210
|
52.242.245.208/28
|
|
@@ -220,6 +232,7 @@ module Legitbot # :nodoc:
|
|
|
220
232
|
68.221.67.224/28
|
|
221
233
|
68.221.67.240/28
|
|
222
234
|
68.221.75.16/28
|
|
235
|
+
70.153.76.16/28
|
|
223
236
|
74.7.35.48/28
|
|
224
237
|
74.7.35.112/28
|
|
225
238
|
74.7.36.64/28
|
|
@@ -241,6 +254,8 @@ module Legitbot # :nodoc:
|
|
|
241
254
|
138.91.30.48/28
|
|
242
255
|
138.91.46.96/28
|
|
243
256
|
168.63.252.240/28
|
|
257
|
+
172.170.8.208/28
|
|
258
|
+
172.171.4.176/28
|
|
244
259
|
172.178.140.144/28
|
|
245
260
|
172.178.141.112/28
|
|
246
261
|
172.178.141.128/28
|
|
@@ -251,11 +266,9 @@ module Legitbot # :nodoc:
|
|
|
251
266
|
172.204.16.64/28
|
|
252
267
|
172.204.27.16/28
|
|
253
268
|
172.212.159.64/28
|
|
269
|
+
172.212.172.160/28
|
|
254
270
|
172.213.11.144/28
|
|
255
|
-
172.213.12.112/28
|
|
256
271
|
172.213.21.16/28
|
|
257
|
-
172.213.21.112/28
|
|
258
|
-
172.213.21.144/28
|
|
259
272
|
172.215.218.96/28
|
|
260
273
|
191.233.1.112/28
|
|
261
274
|
191.233.1.128/28
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Legitbot # :nodoc:
|
|
4
|
+
# https://docs.perplexity.ai/docs/resources/perplexity-crawlers
|
|
5
|
+
class PerplexityBot < BotMatch
|
|
6
|
+
# @fetch:url https://www.perplexity.ai/perplexitybot.json
|
|
7
|
+
# @fetch:jsonpath $.prefixes[*].ipv4Prefix
|
|
8
|
+
ip_ranges %w[
|
|
9
|
+
107.20.236.150/32
|
|
10
|
+
18.210.92.235/32
|
|
11
|
+
18.97.1.228/30
|
|
12
|
+
18.97.9.96/29
|
|
13
|
+
3.211.124.183/32
|
|
14
|
+
3.222.232.239/32
|
|
15
|
+
3.224.62.45/32
|
|
16
|
+
3.231.139.107/32
|
|
17
|
+
]
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# https://docs.perplexity.ai/docs/resources/perplexity-crawlers
|
|
21
|
+
class PerplexityUser < BotMatch
|
|
22
|
+
# @fetch:url https://www.perplexity.ai/perplexity-user.json
|
|
23
|
+
# @fetch:jsonpath $.prefixes[*].ipv4Prefix
|
|
24
|
+
ip_ranges %w[
|
|
25
|
+
18.97.21.0/30
|
|
26
|
+
18.97.43.80/29
|
|
27
|
+
34.193.163.52/32
|
|
28
|
+
44.208.221.197/32
|
|
29
|
+
]
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
rule Legitbot::PerplexityBot, %w[PerplexityBot]
|
|
33
|
+
rule Legitbot::PerplexityUser, %w[Perplexity-User]
|
|
34
|
+
end
|
data/lib/legitbot/version.rb
CHANGED
data/lib/legitbot.rb
CHANGED
|
@@ -18,6 +18,7 @@ require_relative 'legitbot/openai'
|
|
|
18
18
|
require_relative 'legitbot/oracle'
|
|
19
19
|
require_relative 'legitbot/marginalia'
|
|
20
20
|
require_relative 'legitbot/meta'
|
|
21
|
+
require_relative 'legitbot/perplexity'
|
|
21
22
|
require_relative 'legitbot/petalbot'
|
|
22
23
|
require_relative 'legitbot/pinterest'
|
|
23
24
|
require_relative 'legitbot/twitter'
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: legitbot
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.13.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Alexander Azarov
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-05-30 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: fast_interval_tree
|
|
@@ -81,6 +81,7 @@ files:
|
|
|
81
81
|
- ".github/workflows/autocorrect.yml"
|
|
82
82
|
- ".github/workflows/pr.yml"
|
|
83
83
|
- ".github/workflows/release.yml"
|
|
84
|
+
- ".github/workflows/scan-gha.yml"
|
|
84
85
|
- ".gitignore"
|
|
85
86
|
- ".rubocop.yml"
|
|
86
87
|
- ".ruby-version"
|
|
@@ -108,6 +109,7 @@ files:
|
|
|
108
109
|
- lib/legitbot/meta.rb
|
|
109
110
|
- lib/legitbot/openai.rb
|
|
110
111
|
- lib/legitbot/oracle.rb
|
|
112
|
+
- lib/legitbot/perplexity.rb
|
|
111
113
|
- lib/legitbot/petalbot.rb
|
|
112
114
|
- lib/legitbot/pinterest.rb
|
|
113
115
|
- lib/legitbot/twitter.rb
|
|
@@ -156,14 +158,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
156
158
|
requirements:
|
|
157
159
|
- - ">="
|
|
158
160
|
- !ruby/object:Gem::Version
|
|
159
|
-
version: 3.
|
|
161
|
+
version: 3.2.0
|
|
160
162
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
161
163
|
requirements:
|
|
162
164
|
- - ">="
|
|
163
165
|
- !ruby/object:Gem::Version
|
|
164
166
|
version: '0'
|
|
165
167
|
requirements: []
|
|
166
|
-
rubygems_version: 3.
|
|
168
|
+
rubygems_version: 3.5.22
|
|
167
169
|
signing_key:
|
|
168
170
|
specification_version: 4
|
|
169
171
|
summary: Web crawler request validation
|