http_utilities 1.0.2 → 1.0.5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (8) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +4 -3
  3. data/http_utilities.gemspec +2 -2
  4. data/lib/http_utilities/http/proxy_support.rb +9 -7
  5. data/lib/http_utilities/proxies/proxy_checker.rb +48 -13
  6. data/lib/http_utilities/proxies/proxy_module.rb +12 -0
  7. data/lib/http_utilities.rb +1 -1
  8. metadata +35 -21
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: c8fb30e7ce4a8093f1bc2929fc0d94a68e289d40
4
- data.tar.gz: 19cb1986479a7a2ae1957aec32180ecf98c330ac
3
+ metadata.gz: 1c3ac10c88e4301a9be712c06747987b7bbec37e
4
+ data.tar.gz: 85a711b2b5cc059682522bc9c03cd09140afe4a6
5
5
  SHA512:
6
- metadata.gz: 4873c9349bbe2196f1f157ef972e16c0f9241b5e2bf9a67c96c4726b8f4a8a5b4194d207d66198886d4dc7af4ac70f44f152086337fbeba525563d221b138615
7
- data.tar.gz: d2853749c566baf145dfde797ee07c983340d4d83142438bd641cffe3e326ddef73aed208e4d956c46d5f1781b4b34e82e8174a4b9a67ec4c0f5c80c76faf1a0
6
+ metadata.gz: 933e6381c41d5bfd6c0d1e337b936087eaf4e6ea8319caa0ab9878739ecfec1271a596fb551b02d98ada765caa56b5a73dbb8bb3fe337110d09a19bdf8e3a386
7
+ data.tar.gz: b4101592f6b9c6ced152ca5bf933ca74b51cb54bc7c80f486a095612d246a387d6d4ffd23cad4fbfa40f498861d0c6da49d4d6a2efa9f86b6813e92eb90c1a18
data/Gemfile CHANGED
@@ -1,8 +1,9 @@
1
1
  source "http://rubygems.org"
2
2
 
3
- gem "nokogiri", ">= 1.5.5"
4
- gem "mechanize", ">= 2.5"
5
- gem "multi_xml", ">= 0.5"
3
+ gem "nokogiri", ">= 1.5.5"
4
+ gem "mechanize", ">= 2.5"
5
+ gem "multi_xml", ">= 0.5"
6
+ gem "net-ssh", ">= 2.8"
6
7
 
7
8
  gem "activerecord-import", :require => false
8
9
 
data/http_utilities.gemspec CHANGED
@@ -3,10 +3,9 @@ Gem::Specification.new do |s|
3
3
  s.required_rubygems_version = Gem::Requirement.new(">= 1.3.5") if s.respond_to? :required_rubygems_version=
4
4
 
5
5
  s.name = "http_utilities"
6
- s.version = "1.0.2"
6
+ s.version = "1.0.5"
7
7
 
8
8
  s.authors = ["Sebastian Johnsson"]
9
- s.date = "2012-11-22"
10
9
  s.description = "Wrapper for common Http Libraries (Net:HTTP/Open URI/Curl)"
11
10
 
12
11
  s.homepage = "http://github.com/Agiley/http_utilities"
@@ -15,6 +14,7 @@ Gem::Specification.new do |s|
15
14
  s.add_dependency(%q<nokogiri>, [">= 1.5.5"])
16
15
  s.add_dependency(%q<mechanize>, [">= 2.5"])
17
16
  s.add_dependency(%q<multi_xml>, [">= 0.5"])
17
+ s.add_dependency(%q<net-ssh>, [">= 2.8"])
18
18
  s.add_dependency(%q<activerecord-import>, [">= 0"])
19
19
 
20
20
  s.add_development_dependency(%q<rails>, [">= 0"])
data/lib/http_utilities/http/proxy_support.rb CHANGED
@@ -35,13 +35,15 @@ module HttpUtilities
35
35
  self.proxy = specific_proxy
36
36
 
37
37
  elsif (proxy_model_defined?)
38
- proxy_object = Proxy.get_random_proxy(self.proxy[:protocol], self.proxy[:type])
38
+ proxy_object = Proxy.get_random_proxy(self.proxy[:protocol], self.proxy[:type])
39
39
 
40
40
  #log(:info, "[HttpUtilities::Http::ProxySupport] - Randomized Proxy object: #{proxy_object.inspect}")
41
41
 
42
42
  if (proxy_object)
43
43
  self.proxy[:host] = proxy_object.host
44
44
  self.proxy[:port] = proxy_object.port
45
+ proxy_username = proxy_object.username.present? ? proxy_object.username : nil
46
+ proxy_password = proxy_object.password.present? ? proxy_object.password : nil
45
47
  end
46
48
  end
47
49
  end
@@ -52,20 +54,20 @@ module HttpUtilities
52
54
  def set_proxy_credentials(proxy_username, proxy_password, proxy_credentials)
53
55
  if (self.using_proxy? && (!self.proxy[:username] || !self.proxy[:password]))
54
56
  if (proxy_username && proxy_password)
55
- self.proxy[:username] = proxy_username
56
- self.proxy[:password] = proxy_password
57
+ self.proxy[:username] = proxy_username
58
+ self.proxy[:password] = proxy_password
57
59
 
58
60
  elsif (proxy_credentials)
59
61
  if (proxy_credentials.is_a?(Hash))
60
- self.proxy[:username] = proxy_credentials[:username]
61
- self.proxy[:password] = proxy_credentials[:password]
62
+ self.proxy[:username] = proxy_credentials[:username]
63
+ self.proxy[:password] = proxy_credentials[:password]
62
64
 
63
65
  elsif (proxy_credentials.is_a?(String))
64
66
  parts = proxy_credentials.split(":")
65
67
 
66
68
  if (parts && parts.any? && parts.size >= 2)
67
- self.proxy[:username] = parts.first
68
- self.proxy[:password] = parts.second
69
+ self.proxy[:username] = parts.first
70
+ self.proxy[:password] = parts.second
69
71
  end
70
72
  end
71
73
  end
data/lib/http_utilities/proxies/proxy_checker.rb CHANGED
@@ -1,9 +1,12 @@
1
1
  # -*- encoding : utf-8 -*-
2
2
 
3
+ require 'socket'
4
+ require 'net/ssh/proxy/socks5'
5
+ require 'activerecord-import'
6
+
3
7
  module HttpUtilities
4
8
  module Proxies
5
9
  class ProxyChecker
6
- require 'activerecord-import'
7
10
  attr_accessor :client, :processed_proxies
8
11
  attr_accessor :limit, :minimum_successful_attempts, :maximum_failed_attempts
9
12
 
@@ -43,20 +46,52 @@ module HttpUtilities
43
46
  end
44
47
  end
45
48
 
46
- def check_proxy(proxy, timeout = 60)
49
+ def check_proxy(proxy)
50
+ Rails.logger.info "#{Time.now}: Will check if proxy #{proxy.proxy_address} is working."
51
+
52
+ self.send("check_#{proxy.protocol}_proxy")
53
+ end
54
+
55
+ def check_socks_proxy(proxy, test_host: "whois.verisign-grs.com", test_port: 43, test_query: "=google.com")
56
+ valid_proxy = false
57
+
58
+ begin
59
+ socks_proxy = Net::SSH::Proxy::SOCKS5.new(proxy.host, proxy.port, proxy.socks_proxy_credentials)
60
+ client = socks_proxy.open(test_host, test_port)
61
+
62
+ client.write("#{test_query}\r\n")
63
+ response = client.read
64
+
65
+ valid_proxy = (response && response.present?)
66
+
67
+ rescue StandardError => e
68
+ Rails.logger.error "Exception occured while trying to check proxy #{proxy.proxy_address}. Error Class: #{e.class}. Error Message: #{e.message}"
69
+ valid_proxy = false
70
+ end
71
+
72
+ if (valid_proxy)
73
+ Rails.logger.info "#{Time.now}: Proxy #{proxy.proxy_address} is working!"
74
+ else
75
+ Rails.logger.info "#{Time.now}: Proxy #{proxy.proxy_address} is not working!"
76
+ end
77
+
78
+ self.processed_proxies << {proxy: proxy, valid: valid_proxy}
79
+ end
80
+
81
+ def check_http_proxy(proxy, timeout = 60)
47
82
  document = nil
48
83
  valid_proxy = false
49
84
 
50
- options = {:method => :net_http,
51
- :use_proxy => true,
52
- :proxy => proxy.proxy_address,
53
- :proxy_protocol => proxy.protocol,
54
- :timeout => timeout,
55
- :maximum_redirects => 1,
56
- :disable_auth => true
85
+ options = {method: :net_http,
86
+ use_proxy: true,
87
+ proxy: proxy.proxy_address,
88
+ proxy_protocol: proxy.protocol,
89
+ timeout: timeout,
90
+ maximum_redirects: 1,
91
+ disable_auth: true
57
92
  }
58
93
 
59
- Rails.logger.info "#{Time.now}: Fetching Proxy #{proxy.proxy_address}."
94
+ Rails.logger.info "#{Time.now}: Fetching Google.com with proxy #{proxy.proxy_address}."
60
95
 
61
96
  response = self.client.retrieve_parsed_html("http://www.google.com/webhp?hl=en", options)
62
97
 
@@ -73,7 +108,7 @@ module HttpUtilities
73
108
  Rails.logger.info "Title is: #{title}. Proxy #{proxy.proxy_address}"
74
109
 
75
110
  rescue Exception => e
76
- Rails.logger.error "Exception occured while trying to validate proxy. Error Class: #{e.class}. Error Message: #{e.message}"
111
+ Rails.logger.error "Exception occured while trying to check proxy #{proxy.proxy_address}. Error Class: #{e.class}. Error Message: #{e.message}"
77
112
  valid_proxy = false
78
113
  end
79
114
  end
@@ -85,10 +120,10 @@ module HttpUtilities
85
120
  Rails.logger.info "#{Time.now}: Proxy #{proxy.proxy_address} is not working!"
86
121
  end
87
122
 
88
- self.processed_proxies << {:proxy => proxy, :valid => valid_proxy}
123
+ self.processed_proxies << {proxy: proxy, valid: valid_proxy}
89
124
  end
90
125
 
91
- def update_proxies()
126
+ def update_proxies
92
127
  columns = [:host, :port, :last_checked_at, :valid_proxy, :successful_attempts, :failed_attempts]
93
128
  values = []
94
129
 
data/lib/http_utilities/proxies/proxy_module.rb CHANGED
@@ -63,6 +63,18 @@ module HttpUtilities
63
63
  def proxy_credentials
64
64
  return ::Proxy.format_proxy_credentials(self.username, self.password)
65
65
  end
66
+
67
+ def socks_proxy_credentials
68
+ credentials = {}
69
+
70
+ if (!self.username.empty? && !self.password.empty?)
71
+ credentials = {user: self.username, password: self.password}
72
+ elsif (!self.username.empty? && self.password.empty?)
73
+ credentials = {user: self.username}
74
+ end
75
+
76
+ return credentials
77
+ end
66
78
  end
67
79
 
68
80
  end
data/lib/http_utilities.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  # -*- encoding : utf-8 -*-
2
2
  module HttpUtilities
3
- VERSION = "1.0.2"
3
+ VERSION = "1.0.5"
4
4
 
5
5
  require File.join(File.dirname(__FILE__), 'http_utilities/railtie') if defined?(Rails)
6
6
 
metadata CHANGED
@@ -1,125 +1,139 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: http_utilities
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.2
4
+ version: 1.0.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sebastian Johnsson
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2012-11-22 00:00:00.000000000 Z
11
+ date: 2014-06-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: nokogiri
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - '>='
17
+ - - ">="
18
18
  - !ruby/object:Gem::Version
19
19
  version: 1.5.5
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - '>='
24
+ - - ">="
25
25
  - !ruby/object:Gem::Version
26
26
  version: 1.5.5
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: mechanize
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - '>='
31
+ - - ">="
32
32
  - !ruby/object:Gem::Version
33
33
  version: '2.5'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - '>='
38
+ - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '2.5'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: multi_xml
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - '>='
45
+ - - ">="
46
46
  - !ruby/object:Gem::Version
47
47
  version: '0.5'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - '>='
52
+ - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0.5'
55
+ - !ruby/object:Gem::Dependency
56
+ name: net-ssh
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '2.8'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '2.8'
55
69
  - !ruby/object:Gem::Dependency
56
70
  name: activerecord-import
57
71
  requirement: !ruby/object:Gem::Requirement
58
72
  requirements:
59
- - - '>='
73
+ - - ">="
60
74
  - !ruby/object:Gem::Version
61
75
  version: '0'
62
76
  type: :runtime
63
77
  prerelease: false
64
78
  version_requirements: !ruby/object:Gem::Requirement
65
79
  requirements:
66
- - - '>='
80
+ - - ">="
67
81
  - !ruby/object:Gem::Version
68
82
  version: '0'
69
83
  - !ruby/object:Gem::Dependency
70
84
  name: rails
71
85
  requirement: !ruby/object:Gem::Requirement
72
86
  requirements:
73
- - - '>='
87
+ - - ">="
74
88
  - !ruby/object:Gem::Version
75
89
  version: '0'
76
90
  type: :development
77
91
  prerelease: false
78
92
  version_requirements: !ruby/object:Gem::Requirement
79
93
  requirements:
80
- - - '>='
94
+ - - ">="
81
95
  - !ruby/object:Gem::Version
82
96
  version: '0'
83
97
  - !ruby/object:Gem::Dependency
84
98
  name: rspec
85
99
  requirement: !ruby/object:Gem::Requirement
86
100
  requirements:
87
- - - '>='
101
+ - - ">="
88
102
  - !ruby/object:Gem::Version
89
103
  version: '0'
90
104
  type: :development
91
105
  prerelease: false
92
106
  version_requirements: !ruby/object:Gem::Requirement
93
107
  requirements:
94
- - - '>='
108
+ - - ">="
95
109
  - !ruby/object:Gem::Version
96
110
  version: '0'
97
111
  - !ruby/object:Gem::Dependency
98
112
  name: sqlite3
99
113
  requirement: !ruby/object:Gem::Requirement
100
114
  requirements:
101
- - - '>='
115
+ - - ">="
102
116
  - !ruby/object:Gem::Version
103
117
  version: '0'
104
118
  type: :development
105
119
  prerelease: false
106
120
  version_requirements: !ruby/object:Gem::Requirement
107
121
  requirements:
108
- - - '>='
122
+ - - ">="
109
123
  - !ruby/object:Gem::Version
110
124
  version: '0'
111
125
  - !ruby/object:Gem::Dependency
112
126
  name: mysql2
113
127
  requirement: !ruby/object:Gem::Requirement
114
128
  requirements:
115
- - - '>='
129
+ - - ">="
116
130
  - !ruby/object:Gem::Version
117
131
  version: 0.3.11
118
132
  type: :development
119
133
  prerelease: false
120
134
  version_requirements: !ruby/object:Gem::Requirement
121
135
  requirements:
122
- - - '>='
136
+ - - ">="
123
137
  - !ruby/object:Gem::Version
124
138
  version: 0.3.11
125
139
  description: Wrapper for common Http Libraries (Net:HTTP/Open URI/Curl)
@@ -184,17 +198,17 @@ require_paths:
184
198
  - lib
185
199
  required_ruby_version: !ruby/object:Gem::Requirement
186
200
  requirements:
187
- - - '>='
201
+ - - ">="
188
202
  - !ruby/object:Gem::Version
189
203
  version: '0'
190
204
  required_rubygems_version: !ruby/object:Gem::Requirement
191
205
  requirements:
192
- - - '>='
206
+ - - ">="
193
207
  - !ruby/object:Gem::Version
194
208
  version: 1.3.5
195
209
  requirements: []
196
210
  rubyforge_project:
197
- rubygems_version: 2.0.3
211
+ rubygems_version: 2.2.2
198
212
  signing_key:
199
213
  specification_version: 2
200
214
  summary: Wrapper for common Http Libraries (Net:HTTP/Open URI/Curl)