legitbot 1.12.12 → 1.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/pr.yml +1 -1
- data/.ruby-version +1 -1
- data/README.md +1 -0
- data/legitbot.gemspec +1 -1
- data/lib/legitbot/openai.rb +6 -0
- data/lib/legitbot/perplexity.rb +34 -0
- data/lib/legitbot/version.rb +1 -1
- data/lib/legitbot.rb +1 -0
- metadata +5 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 997d2f1450b83a076215f798f9f6d3aad707047ae625e97a42b11f32b0370d53
|
|
4
|
+
data.tar.gz: bcecf8e39cd6040a616539b93813ad59eb63e781e65873a06af8cb09c817aba0
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f8761fac855610b44cc896b04a037b45ebb288205b465699e9759f45b2729a6930f612f29b463f75046a3dcf01823d36bb0e645231837baee5058ba12d8a77b0
|
|
7
|
+
data.tar.gz: 8bd0acc57a5402715ce70e251837db3483777db44f360faced2a0b11b4785d25d660edd381d60d1ae0ad12eea0547adbb99d3636d43913afcf407f5d9e1e430f
|
data/.github/workflows/pr.yml
CHANGED
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
3.
|
|
1
|
+
3.3.11
|
data/README.md
CHANGED
|
@@ -65,6 +65,7 @@ end
|
|
|
65
65
|
- [Oracle Data Cloud Crawler](https://www.oracle.com/corporate/acquisitions/grapeshot/crawler.html)
|
|
66
66
|
- [Marginalia](https://www.marginalia.nu/marginalia-search/for-webmasters/)
|
|
67
67
|
- [Meta / Facebook Web crawlers](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/)
|
|
68
|
+
- [Perplexity](https://docs.perplexity.ai/docs/resources/perplexity-crawlers#ip-address-sources)
|
|
68
69
|
- [Petal search engine](http://aspiegel.com/petalbot)
|
|
69
70
|
- [Pinterest](https://help.pinterest.com/en/articles/about-pinterest-crawler-0)
|
|
70
71
|
- [Twitterbot](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/getting-started),
|
data/legitbot.gemspec
CHANGED
|
@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
|
|
|
19
19
|
'rubygems_mfa_required' => 'true'
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
-
spec.required_ruby_version = '>= 3.
|
|
22
|
+
spec.required_ruby_version = '>= 3.2.0'
|
|
23
23
|
spec.add_dependency 'fast_interval_tree', '~> 0.2', '>= 0.2.2'
|
|
24
24
|
spec.add_dependency 'irrc', '~> 0.2', '>= 0.2.1'
|
|
25
25
|
spec.add_dependency 'logger', '~> 1.7', '>= 1.7.0'
|
data/lib/legitbot/openai.rb
CHANGED
|
@@ -52,6 +52,7 @@ module Legitbot # :nodoc:
|
|
|
52
52
|
4.197.115.112/28
|
|
53
53
|
4.198.72.16/28
|
|
54
54
|
4.205.128.176/28
|
|
55
|
+
4.226.226.32/28
|
|
55
56
|
9.160.163.224/28
|
|
56
57
|
9.160.164.128/28
|
|
57
58
|
13.65.138.96/28
|
|
@@ -74,6 +75,8 @@ module Legitbot # :nodoc:
|
|
|
74
75
|
20.63.221.64/28
|
|
75
76
|
20.97.189.96/28
|
|
76
77
|
20.102.212.144/28
|
|
78
|
+
20.113.218.16/28
|
|
79
|
+
20.113.225.112/28
|
|
77
80
|
20.125.112.224/28
|
|
78
81
|
20.125.144.144/28
|
|
79
82
|
20.161.75.208/28
|
|
@@ -105,6 +108,7 @@ module Legitbot # :nodoc:
|
|
|
105
108
|
20.194.1.0/28
|
|
106
109
|
20.194.157.176/28
|
|
107
110
|
20.198.67.96/28
|
|
111
|
+
20.199.211.160/28
|
|
108
112
|
20.203.245.32/28
|
|
109
113
|
20.204.24.240/28
|
|
110
114
|
20.206.107.192/28
|
|
@@ -157,6 +161,8 @@ module Legitbot # :nodoc:
|
|
|
157
161
|
51.8.155.64/28
|
|
158
162
|
51.8.155.80/28
|
|
159
163
|
51.107.70.192/28
|
|
164
|
+
51.116.2.64/28
|
|
165
|
+
51.116.2.80/28
|
|
160
166
|
52.148.129.32/28
|
|
161
167
|
52.154.22.48/28
|
|
162
168
|
52.156.77.144/28
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Legitbot # :nodoc:
|
|
4
|
+
# https://docs.perplexity.ai/docs/resources/perplexity-crawlers
|
|
5
|
+
class PerplexityBot < BotMatch
|
|
6
|
+
# @fetch:url https://www.perplexity.ai/perplexitybot.json
|
|
7
|
+
# @fetch:jsonpath $.prefixes[*].ipv4Prefix
|
|
8
|
+
ip_ranges %w[
|
|
9
|
+
107.20.236.150/32
|
|
10
|
+
18.210.92.235/32
|
|
11
|
+
18.97.1.228/30
|
|
12
|
+
18.97.9.96/29
|
|
13
|
+
3.211.124.183/32
|
|
14
|
+
3.222.232.239/32
|
|
15
|
+
3.224.62.45/32
|
|
16
|
+
3.231.139.107/32
|
|
17
|
+
]
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# https://docs.perplexity.ai/docs/resources/perplexity-crawlers
|
|
21
|
+
class PerplexityUser < BotMatch
|
|
22
|
+
# @fetch:url https://www.perplexity.ai/perplexity-user.json
|
|
23
|
+
# @fetch:jsonpath $.prefixes[*].ipv4Prefix
|
|
24
|
+
ip_ranges %w[
|
|
25
|
+
18.97.21.0/30
|
|
26
|
+
18.97.43.80/29
|
|
27
|
+
34.193.163.52/32
|
|
28
|
+
44.208.221.197/32
|
|
29
|
+
]
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
rule Legitbot::PerplexityBot, %w[PerplexityBot]
|
|
33
|
+
rule Legitbot::PerplexityUser, %w[Perplexity-User]
|
|
34
|
+
end
|
data/lib/legitbot/version.rb
CHANGED
data/lib/legitbot.rb
CHANGED
|
@@ -18,6 +18,7 @@ require_relative 'legitbot/openai'
|
|
|
18
18
|
require_relative 'legitbot/oracle'
|
|
19
19
|
require_relative 'legitbot/marginalia'
|
|
20
20
|
require_relative 'legitbot/meta'
|
|
21
|
+
require_relative 'legitbot/perplexity'
|
|
21
22
|
require_relative 'legitbot/petalbot'
|
|
22
23
|
require_relative 'legitbot/pinterest'
|
|
23
24
|
require_relative 'legitbot/twitter'
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: legitbot
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.13.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Alexander Azarov
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-05-02 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: fast_interval_tree
|
|
@@ -108,6 +108,7 @@ files:
|
|
|
108
108
|
- lib/legitbot/meta.rb
|
|
109
109
|
- lib/legitbot/openai.rb
|
|
110
110
|
- lib/legitbot/oracle.rb
|
|
111
|
+
- lib/legitbot/perplexity.rb
|
|
111
112
|
- lib/legitbot/petalbot.rb
|
|
112
113
|
- lib/legitbot/pinterest.rb
|
|
113
114
|
- lib/legitbot/twitter.rb
|
|
@@ -156,14 +157,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
156
157
|
requirements:
|
|
157
158
|
- - ">="
|
|
158
159
|
- !ruby/object:Gem::Version
|
|
159
|
-
version: 3.
|
|
160
|
+
version: 3.2.0
|
|
160
161
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
161
162
|
requirements:
|
|
162
163
|
- - ">="
|
|
163
164
|
- !ruby/object:Gem::Version
|
|
164
165
|
version: '0'
|
|
165
166
|
requirements: []
|
|
166
|
-
rubygems_version: 3.
|
|
167
|
+
rubygems_version: 3.5.22
|
|
167
168
|
signing_key:
|
|
168
169
|
specification_version: 4
|
|
169
170
|
summary: Web crawler request validation
|