proxied 0.2.1 → 0.2.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -27,6 +27,7 @@ module Mongoid
27
27
 
28
28
  field :username, type: String
29
29
  field :password, type: String
30
+ field :auth_mode, type: String, default: :credentials
30
31
 
31
32
  field :protocol, type: String, default: :http
32
33
  field :proxy_type, type: String, default: :public
@@ -39,6 +40,9 @@ module Mongoid
39
40
  field :successful_attempts, type: Integer, default: 0
40
41
  field :failed_attempts, type: Integer, default: 0
41
42
 
43
+ field :checkable, type: Boolean, default: true
44
+ field :asyncable, type: Boolean, default: true
45
+
42
46
  field :last_checked_at, type: DateTime
43
47
 
44
48
  # Validations
@@ -1,21 +1,38 @@
1
1
  ::Proxied.configure do |config|
2
- config.proxy_class = "<%= class_name.camelize.to_s %>" # Must be set to the ActiveRecord or Mongoid model that will be used for managing proxies
2
+ # The ActiveRecord or Mongoid model that will be used for managing proxies - Must be set otherwise the gem won't work!
3
+ config.proxy_class = "<%= class_name.camelize.to_s %>"
3
4
 
5
+ # The minimum successful attempts and maximum failed attempts before a proxy shouldn't be considered valid anymore
4
6
  config.minimum_successful_attempts = 1
5
7
  config.maximum_failed_attempts = 10
6
8
 
9
+ # The queue that Sidekiq/ActiveJob will use to check proxies
7
10
  config.job_queue = :proxies
8
11
 
12
+ # Log settings - if Rails is available it will log to the Rails log, otherwise just puts
13
+ config.logger = defined?(Rails) ? -> (message) { Rails.logger.info(message) } : -> (message) { puts(message) }
14
+ config.log_level = :info
15
+
16
+ # The settings below are for configuring the proxy checker service
17
+ config.faraday = {
18
+ adapter: :net_http,
19
+ user_agent: -> { "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.2 Safari/605.1.15" },
20
+ verbose: false
21
+ }
22
+
23
+ # If you're using Typhoeus as adapter, consider setting the following setting either here or in a separate config/initializer:
24
+ # Ethon.logger = Logger.new(nil)
25
+
9
26
  config.http_test = {
10
27
  url: "http://ipinfo.io/ip",
11
- evaluate: -> (proxy, response) { response.to_s.strip.eql?(proxy.ip_address.strip) },
12
- timeout: 10,
28
+ evaluate: -> (proxy, response) { !(response&.to_s&.strip&.downcase =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/i).nil? },
29
+ timeout: 30,
13
30
  }
14
31
 
15
32
  config.socks_test = {
16
33
  hostname: "whois.verisign-grs.com",
17
34
  port: 43,
18
35
  query: "=google.com",
19
- timeout: 10
36
+ timeout: 30
20
37
  }
21
38
  end
@@ -11,13 +11,15 @@ module Proxied
11
11
  self.limit = limit
12
12
  end
13
13
 
14
- def check_proxies(protocol: :all, proxy_type: :all, update: true)
14
+ def check_proxies(protocol: :all, proxy_type: :all, category: nil, update: true)
15
15
  proxies = ::Proxied.configuration.proxy_class.constantize.should_be_checked(
16
- protocol: protocol,
17
- proxy_type: proxy_type,
18
- date: Time.now,
19
- limit: self.limit,
20
- maximum_failed_attempts: self.maximum_failed_attempts
16
+ mode: self.mode.to_sym,
17
+ protocol: protocol,
18
+ proxy_type: proxy_type,
19
+ category: category,
20
+ date: Time.now,
21
+ limit: self.limit,
22
+ maximum_failed_attempts: self.maximum_failed_attempts
21
23
  )
22
24
 
23
25
  if proxies&.any?
@@ -31,56 +33,79 @@ module Proxied
31
33
  ::Proxied::Jobs::CheckProxyJob.perform_async(proxy.id.to_s)
32
34
  end
33
35
  end
34
-
35
36
  else
36
37
  ::Proxied::Logger.log "Couldn't find any proxies to check!"
37
38
  end
38
39
  end
39
-
40
- def check_proxy(proxy, update: true)
40
+
41
+ def check_proxy(proxy, protocol: nil, update: true)
41
42
  ::Proxied::Logger.log "#{Time.now}: Will check if proxy #{proxy.proxy_address} is working."
43
+ protocol ||= proxy.protocol
42
44
 
43
- self.send("check_#{proxy.protocol}_proxy", proxy, update: update)
45
+ self.send("check_#{protocol}_proxy", proxy, update: update)
44
46
  end
45
-
46
- def check_socks_proxy(proxy, test_host: ::Proxied.configuration.socks_test[:hostname], test_port: ::Proxied.configuration.socks_test[:port], test_query: ::Proxied.configuration.socks_test[:query], timeout: ::Proxied.configuration.socks_test[:timeout], update: true)
47
+
48
+ def check_socks_proxy(
49
+ proxy,
50
+ test_host: ::Proxied.configuration.socks_test[:hostname],
51
+ test_port: ::Proxied.configuration.socks_test[:port],
52
+ test_query: ::Proxied.configuration.socks_test[:query],
53
+ timeout: ::Proxied.configuration.socks_test[:timeout],
54
+ update: true
55
+ )
47
56
  valid_proxy = false
48
57
 
49
58
  begin
50
59
  socks_proxy = ::Net::SSH::Proxy::SOCKS5.new(proxy.host, proxy.port, proxy.socks_proxy_credentials)
51
- client = socks_proxy.open(test_host, test_port, {timeout: timeout})
60
+ socket = socks_proxy.open(test_host, test_port, {timeout: timeout})
61
+
62
+ socket.write("#{test_query}\r\n")
63
+ response = socket.read
64
+ valid_proxy = !response.to_s.empty? && response.downcase.include?(test_query.gsub(/^=/, '').downcase)
65
+
66
+ socket.close
52
67
 
53
- client.write("#{test_query}\r\n")
54
- response = client.read
55
-
56
- valid_proxy = !response.to_s.empty?
57
-
58
- client.close
59
-
60
68
  rescue StandardError => e
61
69
  ::Proxied::Logger.log "Exception occured while trying to check proxy #{proxy.proxy_address}. Error Class: #{e.class}. Error Message: #{e.message}"
62
70
  valid_proxy = false
63
71
  end
64
-
72
+
65
73
  update_proxy(proxy, valid_proxy) if update
66
-
74
+
67
75
  return valid_proxy
68
76
  end
69
-
70
- def check_http_proxy(proxy, test_url: ::Proxied.configuration.http_test[:url], evaluate: ::Proxied.configuration.http_test[:evaluate], timeout: ::Proxied.configuration.http_test[:timeout], update: true)
71
- ::Proxied::Logger.log "#{Time.now}: Fetching #{::Proxied.configuration.http_test[:url]} with proxy #{proxy.proxy_address}."
72
-
73
- response = request(test_url, proxy, options: {timeout: timeout})
74
- valid_proxy = evaluate.call(proxy, response)
75
77
 
76
- update_proxy(proxy, valid_proxy) if update
78
+ def check_https_proxy(
79
+ proxy,
80
+ test_url: ::Proxied.configuration.http_test[:url],
81
+ evaluate: ::Proxied.configuration.http_test[:evaluate],
82
+ timeout: ::Proxied.configuration.http_test[:timeout],
83
+ update: true)
77
84
 
85
+ return check_http_proxy(proxy, test_url: test_url, evaluate: evaluate, timeout: timeout, update: update)
86
+ end
87
+
88
+ def check_http_proxy(
89
+ proxy,
90
+ test_url: ::Proxied.configuration.http_test[:url],
91
+ evaluate: ::Proxied.configuration.http_test[:evaluate],
92
+ timeout: ::Proxied.configuration.http_test[:timeout],
93
+ update: true)
94
+
95
+ ::Proxied::Logger.log "#{Time.now}: Fetching #{::Proxied.configuration.http_test[:url]} with proxy #{proxy.proxy_address} (#{proxy.ip_address})."
96
+
97
+ response = request(test_url, proxy, options: {timeout: timeout})
98
+ valid_proxy = evaluate.call(proxy, response)
99
+
100
+ update_proxy(proxy, valid_proxy, response) if update
101
+
78
102
  return valid_proxy
79
103
  end
80
-
81
- def update_proxy(proxy, valid)
82
- ::Proxied::Logger.log "#{Time.now}: Proxy #{proxy.proxy_address} is #{valid ? "working" : "not working"}!"
83
-
104
+
105
+ def update_proxy(proxy, valid, response = nil)
106
+ ::Proxied::Logger.info "#{Time.now}: Proxy #{proxy.proxy_address} (#{proxy.ip_address}) is #{valid ? "working" : "not working"}!"
107
+ ::Proxied::Logger.debug "Response: #{response}" if !valid && response
108
+
84
109
  successful_attempts = proxy.successful_attempts || 0
85
110
  failed_attempts = proxy.failed_attempts || 0
86
111
 
@@ -91,35 +116,48 @@ module Proxied
91
116
  end
92
117
 
93
118
  is_valid = (successful_attempts >= self.minimum_successful_attempts && failed_attempts < self.maximum_failed_attempts)
94
-
119
+
95
120
  proxy.valid_proxy = is_valid
96
121
  proxy.successful_attempts = successful_attempts
97
122
  proxy.failed_attempts = failed_attempts
98
123
  proxy.last_checked_at = Time.now
99
124
  proxy.save
100
125
  end
101
-
126
+
102
127
  private
103
128
  def request(url, proxy, options: {})
104
129
  response = nil
105
130
 
106
- user_agent = options.fetch(:user_agent, ::Proxied.configuration.faraday.fetch(:user_agent, nil))
131
+ user_agent = options.fetch(:user_agent, ::Proxied.configuration.faraday.fetch(:user_agent, nil)&.call)
107
132
  timeout = options.fetch(:timeout, ::Proxied.configuration.http_test[:timeout])
108
-
133
+ auth_mode = proxy.respond_to?(:auth_mode) ? proxy.auth_mode : 'credentials'
134
+
109
135
  begin
110
- response = ::Faraday.new(url: url) do |builder|
136
+ connection = ::Faraday.new(url: url) do |builder|
111
137
  builder.headers["User-Agent"] = user_agent if !user_agent.to_s.empty?
112
138
  builder.options[:timeout] = timeout if timeout
113
139
  builder.proxy = proxy.proxy_options_for_faraday
140
+ #builder.basic_auth(proxy.username, proxy.password) if using_basic_auth?(auth_mode)
114
141
  builder.response :logger if ::Proxied.configuration.verbose_faraday?
115
142
  builder.adapter ::Proxied.configuration.faraday.fetch(:adapter, :net_http)
116
- end.get&.body
117
- rescue Faraday::Error => e
143
+ end
144
+
145
+ response = connection.get&.body
146
+
147
+ rescue Faraday::TimeoutError, Faraday::Error => e
118
148
  ::Proxied::Logger.log "Exception occured while trying to check proxy #{proxy.proxy_address}. Error Class: #{e.class}. Error Message: #{e.message}"
119
149
  end
120
150
 
121
151
  return response
122
152
  end
123
153
 
154
+ def using_credentials?(mode)
155
+ mode.to_s.eql?('credentials')
156
+ end
157
+
158
+ def using_basic_auth?(mode)
159
+ mode.to_s.eql?('basic') || mode.to_s.eql?('basic_auth')
160
+ end
161
+
124
162
  end
125
163
  end
@@ -2,39 +2,45 @@ module Proxied
2
2
  class Configuration
3
3
  attr_accessor :proxy_class
4
4
  attr_accessor :minimum_successful_attempts, :maximum_failed_attempts
5
+ attr_accessor :job_queue
6
+ attr_accessor :logger, :log_level
5
7
  attr_accessor :faraday
6
8
  attr_accessor :http_test, :socks_test
7
- attr_accessor :job_queue
8
- attr_accessor :logger
9
-
10
- def initialize
11
- self.proxy_class = nil # Must be set to the ActiveRecord or Mongoid model that will be used for managing proxies
9
+
10
+ def initialize
11
+ # The ActiveRecord or Mongoid model that will be used for managing proxies - Must be set otherwise the gem won't work!
12
+ self.proxy_class = nil
12
13
 
14
+ # The minimum successful attempts and maximum failed attempts before a proxy shouldn't be considered valid anymore
13
15
  self.minimum_successful_attempts = 1
14
16
  self.maximum_failed_attempts = 10
15
17
 
18
+ # The queue that Sidekiq/ActiveJob will use to check proxies
19
+ self.job_queue = :proxies
20
+
21
+ # Log settings - if Rails is available it will log to the Rails log, otherwise just puts
22
+ self.logger = defined?(Rails) ? -> (message) { Rails.logger.info(message) } : -> (message) { puts(message) }
23
+ self.log_level = :info
24
+
25
+ # The settings below are for configuring the proxy checker service
16
26
  self.faraday = {
17
27
  adapter: :net_http,
18
- user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Safari/605.1.15",
28
+ user_agent: -> { "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.2 Safari/605.1.15" },
19
29
  verbose: false
20
30
  }
21
31
 
22
32
  self.http_test = {
23
- url: "http://ipinfo.io/ip",
24
- evaluate: -> (proxy, response) { response.to_s.strip.eql?(proxy.ip_address.strip) },
25
- timeout: 10,
33
+ url: "https://ipinfo.io/ip",
34
+ evaluate: -> (proxy, response) { !(response&.to_s&.strip&.downcase =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/i).nil? },
35
+ timeout: 30,
26
36
  }
27
37
 
28
38
  self.socks_test = {
29
39
  hostname: "whois.verisign-grs.com",
30
40
  port: 43,
31
41
  query: "=google.com",
32
- timeout: 10
42
+ timeout: 30
33
43
  }
34
-
35
- self.job_queue = :proxies
36
-
37
- self.logger = defined?(Rails) ? -> (message) { Rails.logger.info(message) } : -> (message) { puts(message) }
38
44
  end
39
45
 
40
46
  def verbose_faraday?
@@ -21,19 +21,22 @@ module Proxied
21
21
  ip_address = proxy_item.fetch(:ip_address, host)&.to_s&.strip&.downcase
22
22
 
23
23
  query = {
24
- host: host,
25
- port: port
24
+ host: host,
25
+ port: port
26
26
  }
27
27
 
28
28
  parsed = {
29
- ip_address: ip_address,
30
- protocol: proxy_item.fetch(:protocol, "http")&.to_s&.strip&.downcase,
31
- proxy_type: proxy_item.fetch(:type, :private)&.to_s&.strip,
32
- category: proxy_item.fetch(:category, nil)&.to_s&.strip&.downcase,
33
- country: proxy_item.fetch(:country, nil)&.to_s&.strip&.upcase,
34
- city: proxy_item.fetch(:city, nil)&.to_s&.strip&.downcase,
35
- username: proxy_item.fetch(:username, nil)&.to_s&.strip,
36
- password: proxy_item.fetch(:password, nil)&.to_s&.strip
29
+ ip_address: ip_address,
30
+ protocol: proxy_item.fetch(:protocol, "http")&.to_s&.strip&.downcase,
31
+ proxy_type: proxy_item.fetch(:type, :private)&.to_s&.strip,
32
+ category: proxy_item.fetch(:category, nil)&.to_s&.strip&.downcase,
33
+ country: proxy_item.fetch(:country, nil)&.to_s&.strip&.upcase,
34
+ city: proxy_item.fetch(:city, nil)&.to_s&.strip&.downcase,
35
+ username: proxy_item.fetch(:username, nil)&.to_s&.strip,
36
+ password: proxy_item.fetch(:password, nil)&.to_s&.strip,
37
+ auth_mode: proxy_item.fetch(:auth_mode, :credentials)&.to_s&.strip,
38
+ checkable: proxy_item.fetch(:checkable, true),
39
+ asyncable: proxy_item.fetch(:asyncable, true),
37
40
  }.merge(query)
38
41
 
39
42
  proxy = ::Proxied.configuration.proxy_class.constantize.where(query).first || ::Proxied.configuration.proxy_class.constantize.new
@@ -1,9 +1,19 @@
1
1
  module Proxied
2
2
  class Logger
3
+ class << self
4
+
5
+ def info(message)
6
+ log(message) if ::Proxied.configuration.log_level.to_sym.eql?(:info)
7
+ end
3
8
 
4
- def self.log(message)
5
- ::Proxied.configuration.logger.call(message)
6
- end
9
+ def debug(message)
10
+ log(message) if ::Proxied.configuration.log_level.to_sym.eql?(:debug)
11
+ end
12
+
13
+ def log(message)
14
+ ::Proxied.configuration.logger.call(message)
15
+ end
7
16
 
17
+ end
8
18
  end
9
19
  end
@@ -11,8 +11,10 @@ module Proxied
11
11
  end
12
12
 
13
13
  module ClassMethods
14
- def should_be_checked(protocol: :all, proxy_type: :all, date: Time.now, limit: 10, maximum_failed_attempts: 10)
15
- proxies = get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
14
+ def should_be_checked(mode: :synchronous, protocol: :all, proxy_type: :all, category: nil, country: nil, date: Time.now, limit: nil, maximum_failed_attempts: 10)
15
+ proxies = get_proxies(protocol: protocol, proxy_type: proxy_type, category: category, country: country)
16
+ proxies = proxies.where(checkable: true)
17
+ proxies = proxies.where(asyncable: true) if mode.to_sym.eql?(:asynchronous)
16
18
 
17
19
  proxies = proxies.any_of(
18
20
  {:last_checked_at.exists => false},
@@ -27,16 +29,15 @@ module Proxied
27
29
  )
28
30
 
29
31
  proxies = proxies.order_by([[:valid_proxy, :asc], [:failed_attempts, :asc], [:last_checked_at, :asc]])
30
- proxies = proxies.limit(limit)
32
+ proxies = proxies.limit(limit) if limit && !limit.zero?
31
33
 
32
34
  return proxies
33
35
  end
34
36
 
35
- def get_valid_proxies(protocol: :all, proxy_type: :all, category: nil, maximum_failed_attempts: nil)
36
- proxies = get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
37
+ def get_valid_proxies(protocol: :all, proxy_type: :all, category: nil, country: nil, maximum_failed_attempts: nil)
38
+ proxies = get_proxies(protocol: protocol, proxy_type: proxy_type, category: category, country: country)
37
39
  proxies = proxies.where(valid_proxy: true)
38
40
  proxies = proxies.where(:failed_attempts.lte => maximum_failed_attempts) if maximum_failed_attempts
39
- proxies = proxies.where(category: category) unless category.to_s.empty?
40
41
  end
41
42
 
42
43
  def get_random_proxy(protocol: :all, proxy_type: :all, category: nil, maximum_failed_attempts: nil, retries: 3)
@@ -2,17 +2,19 @@ module Proxied
2
2
  module Shared
3
3
 
4
4
  module ClassMethods
5
- def get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
5
+ def get_proxies(protocol: :all, proxy_type: :all, category: nil, country: nil)
6
6
  proxies = ::Proxy.where(nil)
7
- proxies = proxies.where(protocol: protocol) if (protocol && !protocol.downcase.to_sym.eql?(:all))
8
- proxies = proxies.where(proxy_type: proxy_type) if (proxy_type && !proxy_type.downcase.to_sym.eql?(:all))
7
+ proxies = proxies.where(protocol: protocol) if (protocol && !protocol.downcase.to_sym.eql?(:all))
8
+ proxies = proxies.where(proxy_type: proxy_type) if (proxy_type && !proxy_type.downcase.to_sym.eql?(:all))
9
+ proxies = proxies.where(category: category) unless category.to_s.empty?
10
+ proxies = proxies.where(country: country.to_s.upcase) unless country.to_s.empty?
9
11
 
10
12
  return proxies
11
13
  end
12
14
 
13
15
  # Keep these for compatibility for now, they just wrap the utility functions
14
- def format_proxy_address(host, port = 80, include_http = false)
15
- ::Proxied::Utilities.format_proxy_address(host: host, port: port, include_http: include_http)
16
+ def format_proxy_address(host, port = 80, include_protocol = false)
17
+ ::Proxied::Utilities.format_proxy_address(host: host, port: port, include_protocol: include_protocol)
16
18
  end
17
19
 
18
20
  def format_proxy_credentials(username, password)
@@ -21,8 +23,13 @@ module Proxied
21
23
  end
22
24
 
23
25
  module InstanceMethods
24
- def proxy_address(include_http: false)
25
- ::Proxied::Utilities.format_proxy_address(host: self.host, port: self.port, include_http: include_http)
26
+ def proxy_address(include_protocol: false)
27
+ case self.auth_mode.to_sym
28
+ when :credentials
29
+ ::Proxied::Utilities.format_proxy_address(host: self.host, port: self.port, protocol: self.protocol, include_protocol: include_protocol)
30
+ when :url
31
+ ::Proxied::Utilities.format_proxy_address(host: self.host, port: self.port, protocol: self.protocol, username: self.username, password: self.password, include_protocol: include_protocol)
32
+ end
26
33
  end
27
34
 
28
35
  def proxy_credentials
@@ -34,12 +41,16 @@ module Proxied
34
41
  end
35
42
 
36
43
  def proxy_options_for_faraday
37
- ::Proxied::Utilities.proxy_options_for_faraday(host: self.host, port: self.port, username: self.username, password: self.password)
44
+ ::Proxied::Utilities.proxy_options_for_faraday(
45
+ host: self.host,
46
+ port: self.port,
47
+ protocol: self.protocol,
48
+ username: self.username,
49
+ password: self.password,
50
+ auth_mode: self.auth_mode
51
+ )
38
52
  end
39
53
 
40
- def proxy_switcher_import_format
41
- ::Proxied::Utilities.proxy_switcher_import_format(host: self.host, port: self.port, username: self.username, password: self.password, country: self.country)
42
- end
43
54
  end
44
55
 
45
56
  end