legitbot 0.2.6 → 0.2.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +4 -1
  3. data/legitbot.gemspec +3 -3
  4. data/lib/legitbot/facebook.rb +16 -7
  5. data/lib/legitbot/version.rb +1 -1
  6. data/test/facebook_test.rb +15 -0
  7. metadata +7 -7
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b14d74ce5edae0b0422d2c094d6ad2a90fc27b82437985dea65f627800db95aa
4
- data.tar.gz: b1834d8450279a73203adf3af6876bda25fc61c2c4f82fc0a9de766918002573
3
+ metadata.gz: ae1f2268a473ad5c04afb57f84e947bface5de9fc0b126639ecd9d492f1fefa0
4
+ data.tar.gz: 315a78cf41895bf503de265c77774152d944832959d22c9759576080b5d8d390
5
5
  SHA512:
6
- metadata.gz: f7b7ca927a817ed0360751954245b3a85f5f2cc00affcf55a1da459ee719d82f910239f37ac0ccfabe7e44d477c340edbf5a6bcba1c01edde4c3e0f17f35345c
7
- data.tar.gz: 8d87d281b096eeb42675f92080fc3fefbf6c3c8fc980b7f62cb3ae3fad856310a1fde519efd6cb7420644e4408006e4831f8142131b277de548f4de2590d76df
6
+ metadata.gz: 5caded01279b6e06f3979ea29722b55981d54b69e692a4da27e1f818704dc8c6c486353dfff8723c5afdad241950072047a4ffaf4105f17eb38f32e44fa95836
7
+ data.tar.gz: 2d9f49178129bf04ee6112d6f3f8eca8ca2fb2878daecb26a76b38ea28de63befca2744ecc5087b2a37b09e184621cde8a0016a6513be5556df7fc7e907cb3ad
data/.travis.yml CHANGED
@@ -4,6 +4,9 @@ cache: bundler
4
4
  rvm:
5
5
  - 2.5
6
6
  - 2.3
7
- - jruby
7
+ - jruby-head
8
8
  jdk:
9
9
  - openjdk8
10
+ before_install:
11
+ - gem update --system
12
+ - gem install bundler
data/legitbot.gemspec CHANGED
@@ -10,13 +10,13 @@ Gem::Specification.new do |spec|
10
10
  spec.author = "Alexander Azarov"
11
11
  spec.email = "self@alaz.me"
12
12
  spec.homepage = "https://github.com/alaz/legitbot"
13
- spec.summary = %q{Validate Web request was made by legitimate search engine}
13
+ spec.summary = %q{Validate requests from Web crawlers: impersonating or not?}
14
14
  spec.description = "A library to make sure a Web request has been "\
15
- "made by a real search engine, not a fake"
15
+ "made by a real search engine, not a malicious agent"
16
16
 
17
17
  spec.required_ruby_version = '>= 2.3.0'
18
18
  spec.add_dependency "irrc", ">= 0.2.1"
19
- spec.add_dependency "segment_tree"
19
+ spec.add_dependency "augmented_interval_tree", ">= 0.1.1"
20
20
  spec.add_development_dependency "rake"
21
21
  spec.add_development_dependency "minitest"
22
22
 
data/lib/legitbot/facebook.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  require 'ipaddr'
2
2
  require 'irrc'
3
- require 'segment_tree'
3
+ require 'interval_tree'
4
4
 
5
5
  module Legitbot
6
6
  # https://developers.facebook.com/docs/sharing/webmasters/crawler
@@ -10,7 +10,7 @@ module Legitbot
10
10
 
11
11
  def valid?
12
12
  ip = IPAddr.new(@ip)
13
- Facebook.valid_ips[ip.ipv4? ? :ipv4 : :ipv6].find(ip)
13
+ Facebook.valid_ips[ip.ipv4? ? :ipv4 : :ipv6].search(ip.to_i).size > 0
14
14
  end
15
15
 
16
16
  @mutex = Mutex.new
@@ -24,15 +24,24 @@ module Legitbot
24
24
  end
25
25
 
26
26
  def self.load_ips
27
+ whois.map do |(family, records)|
28
+ ranges = records.map do |cidr|
29
+ range = IPAddr.new(cidr).to_range
30
+ (range.begin.to_i..range.end.to_i)
31
+ end
32
+ [family, IntervalTree::Tree.new(ranges)]
33
+ end.to_h
34
+ end
35
+
36
+ def self.whois
27
37
  client = Irrc::Client.new
28
38
  client.query :radb, AS
29
39
  results = client.perform
30
40
 
31
- Hash[%i(ipv4 ipv6).map { |k|
32
- [k, SegmentTree.new(results[AS][k][AS].map { |cidr|
33
- [IPAddr.new(cidr).to_range, true]
34
- })]
35
- }]
41
+ %i(ipv4 ipv6).map do |family|
42
+ [family, results[AS][family][AS]]
43
+ end.to_h
44
+ # { ipv4: results[AS][:ipv4][AS], ipv6: results[AS][:ipv6][AS] }
36
45
  end
37
46
  end
38
47
 
data/lib/legitbot/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Legitbot
2
- VERSION = '0.2.6'
2
+ VERSION = '0.2.7'
3
3
  end
data/test/facebook_test.rb CHANGED
@@ -1,11 +1,26 @@
1
1
  require 'minitest/autorun'
2
2
  require 'legitbot'
3
3
 
4
+ module Legitbot
5
+ class Facebook
6
+ def self.whois
7
+ {
8
+ ipv4: ["69.63.176.0/20", "66.220.144.0/20", "66.220.144.0/21", "69.63.184.0/21", "69.63.176.0/21", "74.119.76.0/22", "69.171.255.0/24", "173.252.64.0/18", "69.171.224.0/19", "69.171.224.0/20", "103.4.96.0/22", "69.63.176.0/24", "173.252.64.0/19", "173.252.70.0/24", "31.13.64.0/18", "31.13.24.0/21", "66.220.152.0/21", "66.220.159.0/24", "69.171.239.0/24", "69.171.240.0/20", "31.13.64.0/19", "31.13.64.0/24", "31.13.65.0/24", "31.13.67.0/24", "31.13.68.0/24", "31.13.69.0/24", "31.13.70.0/24", "31.13.71.0/24", "31.13.72.0/24", "31.13.73.0/24", "31.13.74.0/24", "31.13.75.0/24", "31.13.76.0/24", "31.13.77.0/24", "31.13.96.0/19", "31.13.66.0/24", "173.252.96.0/19", "69.63.178.0/24", "31.13.78.0/24", "31.13.79.0/24", "31.13.80.0/24", "31.13.82.0/24", "31.13.83.0/24", "31.13.84.0/24", "31.13.85.0/24", "31.13.86.0/24", "31.13.87.0/24", "31.13.88.0/24", "31.13.89.0/24", "31.13.90.0/24", "31.13.91.0/24", "31.13.92.0/24", "31.13.93.0/24", "31.13.94.0/24", "31.13.95.0/24", "69.171.253.0/24", "69.63.186.0/24", "31.13.81.0/24", "179.60.192.0/22", "179.60.192.0/24", "179.60.193.0/24", "179.60.194.0/24", "179.60.195.0/24", "185.60.216.0/22", "45.64.40.0/22", "185.60.216.0/24", "185.60.217.0/24", "185.60.218.0/24", "185.60.219.0/24", "129.134.0.0/16", "157.240.0.0/16", "157.240.8.0/24", "157.240.0.0/24", "157.240.1.0/24", "157.240.2.0/24", "157.240.3.0/24", "157.240.4.0/24", "157.240.5.0/24", "157.240.6.0/24", "157.240.7.0/24", "157.240.9.0/24", "157.240.10.0/24", "157.240.16.0/24", "157.240.19.0/24", "157.240.11.0/24", "157.240.12.0/24", "157.240.13.0/24", "157.240.14.0/24", "157.240.15.0/24", "157.240.17.0/24", "157.240.18.0/24", "157.240.20.0/24", "157.240.21.0/24", "157.240.22.0/24", "157.240.23.0/24", "157.240.0.0/17", "69.171.250.0/24", "157.240.24.0/24", "157.240.25.0/24", "199.201.64.0/24", "199.201.65.0/24", "199.201.64.0/22", "204.15.20.0/22", "157.240.192.0/24", "129.134.0.0/17", "157.240.198.0/24"],
9
+ ipv6: []
10
+ }
11
+ end
12
+ end
13
+ end
14
+
4
15
  class FacebookTest < Minitest::Test
5
16
  def test_valid_ip
6
17
  ip = "69.63.186.89"
7
18
  match = Legitbot::Facebook.new(ip)
8
19
  assert match.valid?, msg: "#{ip} is a valid Facebook IP"
20
+
21
+ ip = '69.171.251.1'
22
+ match = Legitbot::Facebook.new(ip)
23
+ assert match.valid?, msg: "#{ip} is a valid Facebook IP"
9
24
  end
10
25
 
11
26
  def test_invalid_ip
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: legitbot
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.6
4
+ version: 0.2.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - Alexander Azarov
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-03-14 00:00:00.000000000 Z
11
+ date: 2019-04-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: irrc
@@ -25,19 +25,19 @@ dependencies:
25
25
  - !ruby/object:Gem::Version
26
26
  version: 0.2.1
27
27
  - !ruby/object:Gem::Dependency
28
- name: segment_tree
28
+ name: augmented_interval_tree
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - ">="
32
32
  - !ruby/object:Gem::Version
33
- version: '0'
33
+ version: 0.1.1
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
- version: '0'
40
+ version: 0.1.1
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: rake
43
43
  requirement: !ruby/object:Gem::Requirement
@@ -67,7 +67,7 @@ dependencies:
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0'
69
69
  description: A library to make sure a Web request has been made by a real search engine,
70
- not a fake
70
+ not a malicious agent
71
71
  email: self@alaz.me
72
72
  executables: []
73
73
  extensions: []
@@ -123,7 +123,7 @@ rubyforge_project:
123
123
  rubygems_version: 2.7.6.2
124
124
  signing_key:
125
125
  specification_version: 4
126
- summary: Validate Web request was made by legitimate search engine
126
+ summary: 'Validate requests from Web crawlers: impersonating or not?'
127
127
  test_files:
128
128
  - test/legitbot_test.rb
129
129
  - test/pinterest_test.rb