proxied 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 29acafb1d9f1d68dd0c3992e3948516fc939fb351a521c819932417c7bc68cfb
-  data.tar.gz: 60a91e0f65fe7a4c2468a0c02136ace0184480daa2255fcb44795f7088aea214
+  metadata.gz: 517de1167f3ffecb00a5928fc7a9372fb4190ac7cfa4c6ab61c9ba623c59dbb7
+  data.tar.gz: 4bddfb8d6887527f1319b7ed653d2519cad672b697c76e87a9658bb673bbfc88
 SHA512:
-  metadata.gz: 5421ffe9184290f0becd77ae161d000f3a2f997fc9dd109efab7d365dfd49e8fd6e42c3aed2cb19e7e0142158be0d2ab16e545ea676a6f54a8a74bd3c683faaf
-  data.tar.gz: 1485343312be1dc6204be61a839cfc9ff45ad35c23f94f4ea2352411922100ebad413191aee04cc26139f135a6af668e50b1da28086d325587f0bf19530a8901
+  metadata.gz: 7a689a4a001970b1ae2f2a7d886b4406421a4df8e887ee40386f22f6de1e97c0167c1481ae9a4e61d52a341d0624ac6d92d684700190bda3486c7cf47e589cbd
+  data.tar.gz: c26a67137820c580253bb7dc9e9d8ee683ea33fbccda247cd56c5b48c914a71a2d176e3dc79099a2a24d7b55f84860593afa15b860c4df183daa89f439de726c

data/.gitignore

@@ -17,3 +17,10 @@
 
 # appraisals
 /gemfiles/*.lock
+
+# RVM
+.ruby-version
+.ruby-gemset
+
+# DS_Store
+.DS_Store

data/Gemfile.lock

@@ -2,8 +2,8 @@ PATH
   remote: .
   specs:
     proxied (0.1.0)
-      faraday (~> 0.15.2)
-      net-ssh (~> 5.0.2)
+      faraday (>= 0.14)
+      net-ssh (>= 4.0)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -24,6 +24,15 @@ Or install it yourself as:
 
     $ gem install proxied
 
+Generate the initializer for your Rails app:
+
+    $ rails generate proxied:install MODEL
+
+Generate the migration and model:
+
+    $ rails generate proxied MODEL
+
+
 ## Usage
 
 - TODO -

data/lib/generators/active_record/templates/migration.rb

@@ -3,8 +3,8 @@
 class ProxiedCreate<%= table_name.camelize %> < ActiveRecord::Migration<%= migration_version %>
   def change
     create_table :<%= table_name %><%= primary_key_type %> do |t|
-      t.string  :host,                  null: false
-      t.integer :port,                  null: false
+      t.string  :host,                  null: false, index: true
+      t.integer :port,                  null: false, index: true
       t.string  :username
       t.string  :password
       
@@ -12,6 +12,9 @@ class ProxiedCreate<%= table_name.camelize %> < ActiveRecord::Migration<%= migra
       t.string  :proxy_type,            null: false, default: 'public', index: true
       t.string  :category
       
+      t.string  :country,               index: true
+      t.string  :city,                  index: true
+      
       t.datetime :last_checked_at,      index: true
       
       t.boolean  :valid_proxy,          null:  false, default: false, index: true

data/lib/generators/mongoid/proxied_generator.rb

@@ -31,6 +31,9 @@ module Mongoid
   field :proxy_type, type: String, default: :public
   field :category, type: String
   
+  field :country, type: String
+  field :city, type: String
+  
   field :valid_proxy, type: Boolean, default: false
   field :successful_attempts, type: Integer, default: 0
   field :failed_attempts, type: Integer, default: 0

data/lib/generators/templates/proxied.rb

@@ -1,26 +1,8 @@
 ::Proxied.configure do |config|
-  config.proxy_class                  =   <%= class_name.camelize.to_s %> # Must be set to the ActiveRecord or Mongoid model that will be used for managing proxies
+  config.proxy_class                  =   "<%= class_name.camelize.to_s %>" # Must be set to the ActiveRecord or Mongoid model that will be used for managing proxies
   
   config.minimum_successful_attempts  =   1
   config.maximum_failed_attempts      =   10
   
-  config.faraday          =   {
-    adapter:    :net_http,
-    user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Safari/605.1.15",
-    verbose:    false
-  }
-  
-  config.http_test        =   {
-    url:      "http://www.google.com/robots.txt",
-    timeout:  10,
-  }
-  
-  config.socks_test       =   {
-    hostname: "whois.verisign-grs.com",
-    port:     43,
-    query:    "=google.com",
-    timeout:  10
-  }
-  
-  config.logger           =   defined?(Rails) ? -> (message) { Rails.logger.info(message) } : -> (message) { puts(message) }
+  config.job_queue                    =   :proxies
 end

data/lib/proxied.rb

@@ -1,3 +1,21 @@
+module Proxied
+  class << self
+    attr_writer :configuration
+  end
+
+  def self.configuration
+    @configuration ||= ::Proxied::Configuration.new
+  end
+
+  def self.reset
+    @configuration = ::Proxied::Configuration.new
+  end
+
+  def self.configure
+    yield(configuration)
+  end
+end
+
 # Gems
 require "faraday"
 require "net/ssh/proxy/socks5"
@@ -23,21 +41,3 @@ if defined?(Sidekiq)
 end
 
 require "proxied/railtie" if defined?(Rails)
-
-module Proxied
-  class << self
-    attr_writer :configuration
-  end
-
-  def self.configuration
-    @configuration ||= ::Proxied::Configuration.new
-  end
-
-  def self.reset
-    @configuration = ::Proxied::Configuration.new
-  end
-
-  def self.configure
-    yield(configuration)
-  end
-end

data/lib/proxied/checker.rb

@@ -9,8 +9,8 @@ module Proxied
       self.limit                        =   limit
     end
 
-    def check_proxies(protocol: :all, proxy_type: :all, mode: :synchronous)
-      proxies                           =   ::Proxied.configuration.proxy_class.should_be_checked(
+    def check_proxies(protocol: :all, proxy_type: :all, mode: :synchronous, update: true)
+      proxies                           =   ::Proxied.configuration.proxy_class.constantize.should_be_checked(
         protocol:                 protocol,
         proxy_type:               proxy_type,
         date:                     Time.now,
@@ -24,7 +24,7 @@ module Proxied
         proxies.each do |proxy|
           case mode
             when :synchronous
-              check_proxy(proxy)
+              check_proxy(proxy, update: update)
             when :sidekiq
               ::Proxied::Jobs::CheckProxyJob.perform_async(proxy.id.to_s)
           end

data/lib/proxied/configuration.rb

@@ -4,6 +4,7 @@ module Proxied
     attr_accessor :minimum_successful_attempts, :maximum_failed_attempts
     attr_accessor :faraday
     attr_accessor :http_test, :socks_test
+    attr_accessor :job_queue
     attr_accessor :logger
   
     def initialize      
@@ -30,6 +31,8 @@ module Proxied
         timeout:  10
       }
       
+      self.job_queue        =   :proxies
+      
       self.logger           =   defined?(Rails) ? -> (message) { Rails.logger.info(message) } : -> (message) { puts(message) }
     end
     

data/lib/proxied/jobs/check_proxies_job.rb

@@ -2,10 +2,10 @@ module Proxied
   module Jobs
     class CheckProxiesJob
       include ::Sidekiq::Worker
-      sidekiq_options queue: :proxies
+      sidekiq_options queue: ::Proxied.configuration.job_queue
 
       def perform(protocol = :all, proxy_type = :all, mode = :synchronous)
-        ::Proxied::Checker.new.check_proxies(protocol: protocol.to_sym, proxy_type: proxy_type.to_sym, mode: mode.to_sym)
+        ::Proxied::Checker.new.check_proxies(protocol: protocol.to_sym, proxy_type: proxy_type.to_sym, mode: mode.to_sym, update: true)
       end
     end
   end

data/lib/proxied/jobs/check_proxy_job.rb

@@ -2,15 +2,14 @@ module Proxied
   module Jobs
     class CheckProxyJob
       include ::Sidekiq::Worker
-      sidekiq_options queue: :proxies
+      sidekiq_options queue: ::Proxied.configuration.job_queue
 
       def perform(proxy_id)
         proxy_object  =   ::Proxy.where(id: proxy_id).first
 
         if proxy_object
           checker     =   ::Proxied::Checker.new
-          checker.check_proxy(proxy_object)
-          checker.update_proxies
+          checker.check_proxy(proxy_object, update: true)
         end
       end
     end

data/lib/proxied/seeder.rb

@@ -0,0 +1,45 @@
+module Proxied
+  class Seeder
+    
+    # Expected format:
+    # {
+    #   host
+    #   port
+    #   protocol (http,socks)
+    #   proxy_type (private,public)
+    #   category
+    #   username
+    #   password
+    # }
+    
+    def import(proxies)
+      proxies.each do |proxy_item|
+        query                       =   {
+          host:       proxy_item[:ip_address]&.to_s&.strip,
+          port:       proxy_item[:port]&.to_s&.strip&.to_i
+        }
+    
+        parsed                      =   {
+          protocol:   proxy_item.fetch(:protocol, "http")&.to_s&.strip&.downcase,
+          proxy_type: proxy_item.fetch(:type, :private)&.to_s&.strip,
+          category:   proxy_item.fetch(:category, nil)&.to_s&.strip&.downcase,
+          username:   proxy_item.fetch(:username, nil)&.to_s&.strip,
+          password:   proxy_item.fetch(:password, nil)&.to_s&.strip
+        }.merge(query)
+    
+        proxy                       =   ::Proxied.configuration.proxy_class.constantize.where(query).first || ::Proxied.configuration.proxy_class.constantize.new
+    
+        parsed.each do |key, value|
+          proxy.send("#{key}=", value)
+        end
+    
+        proxy.last_checked_at       =   Time.now
+        proxy.valid_proxy           =   true
+        proxy.successful_attempts   =   1
+    
+        proxy.save
+      end
+    end
+    
+  end
+end

data/lib/proxied/sql/proxy_methods.rb

@@ -4,10 +4,10 @@ module Proxied
     module ProxyMethods
     
       def self.included(base)
-        base.send :extend ::Proxied::Shared::ClassMethods
-        base.send :extend, ClassMethods
+        base.extend(::Proxied::Shared::ClassMethods)
+        base.extend(ClassMethods)
         
-        base.send :include ::Proxied::Shared::InstanceMethods
+        base.include(::Proxied::Shared::InstanceMethods)
       end
 
       module ClassMethods

data/lib/proxied/version.rb

@@ -1,3 +1,3 @@
 module Proxied
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/proxied.gemspec

@@ -23,8 +23,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
   
-  spec.add_dependency "faraday",                    "~> 0.15.2"
-  spec.add_dependency "net-ssh",                    "~> 5.0.2"
+  spec.add_dependency "faraday",                    ">= 0.14"
+  spec.add_dependency "net-ssh",                    ">= 4.0"
   
   spec.add_development_dependency "bundler",        "~> 1.16.2"
   spec.add_development_dependency "rake",           "~> 12.3.1"

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: proxied
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Sebastian
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-06-29 00:00:00.000000000 Z
+date: 2018-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.15.2
+        version: '0.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.15.2
+        version: '0.14'
 - !ruby/object:Gem::Dependency
   name: net-ssh
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 5.0.2
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 5.0.2
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -268,8 +268,6 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
-- ".ruby-gemset"
-- ".ruby-version"
 - ".travis.yml"
 - Appraisals
 - CODE_OF_CONDUCT.md
@@ -283,7 +281,6 @@ files:
 - gemfiles/.bundle/config
 - gemfiles/activerecord5.gemfile
 - gemfiles/mongoid7.gemfile
-- lib/.DS_Store
 - lib/generators/active_record/proxied_generator.rb
 - lib/generators/active_record/templates/migration.rb
 - lib/generators/mongoid/proxied_generator.rb
@@ -292,7 +289,6 @@ files:
 - lib/generators/proxied/proxied_generator.rb
 - lib/generators/templates/proxied.rb
 - lib/proxied.rb
-- lib/proxied/.DS_Store
 - lib/proxied/checker.rb
 - lib/proxied/configuration.rb
 - lib/proxied/jobs/check_proxies_job.rb
@@ -300,6 +296,7 @@ files:
 - lib/proxied/logger.rb
 - lib/proxied/nosql/proxy_methods.rb
 - lib/proxied/railtie.rb
+- lib/proxied/seeder.rb
 - lib/proxied/shared.rb
 - lib/proxied/sql/proxy_methods.rb
 - lib/proxied/version.rb

data/.ruby-gemset

@@ -1 +0,0 @@
-proxied

data/.ruby-version

@@ -1 +0,0 @@
-2.6.0-preview2