active_proxy 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 8bd1368da1b34144f5d7878a5937a56cea5cc1664cc485f4f90a66c10d2d1e95
+  data.tar.gz: b48f92261b395ff5d367c1171e200b705d515f0923c9ed4218db8e9384462d24
+SHA512:
+  metadata.gz: 5ffbd1994514e253d65a1524ca3f0fa145a0d57e614884668994b3df7a3fa1af0a191583967fa3f2b7f2ec58fbfa11423b2d0077145aa6669fb6477434571441
+  data.tar.gz: 722b26f35ecf28a9bfc457adba21fe8528e375775ffc309612599f676f8a89d3cb06ee0efda900c7f0e29cf1e5563ad5a8ada07db945bb523fd02f0b6fe8f8a2

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.idea/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,6 @@
+inherit_gem:
+  rubocop-rails_config:
+    - config/rails.yml
+
+AllCops:
+  TargetRubyVersion: 2.5

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.5.3
+before_install: gem install bundler -v 1.16.6

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in active_proxy.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,152 @@
+PATH
+  remote: .
+  specs:
+    active_proxy (1.0.0)
+      nokogiri (>= 1.6.8)
+      proxy_fetcher (>= 0.9.0)
+      retries (>= 0.0.5)
+      user-agent-randomizer (>= 0.2.0)
+      zeitwerk (>= 1.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actionpack (5.2.2)
+      actionview (= 5.2.2)
+      activesupport (= 5.2.2)
+      rack (~> 2.0)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    actionview (5.2.2)
+      activesupport (= 5.2.2)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.3)
+    activesupport (5.2.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    addressable (2.6.0)
+      public_suffix (>= 2.0.2, < 4.0)
+    ast (2.4.0)
+    builder (3.2.3)
+    concurrent-ruby (1.1.4)
+    crass (1.0.4)
+    diff-lcs (1.3)
+    domain_name (0.5.20180417)
+      unf (>= 0.0.5, < 1.0.0)
+    erubi (1.8.0)
+    ethon (0.12.0)
+      ffi (>= 1.3.0)
+    ffi (1.10.0)
+    http (3.3.0)
+      addressable (~> 2.3)
+      http-cookie (~> 1.0)
+      http-form_data (~> 2.0)
+      http_parser.rb (~> 0.6.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    http-form_data (2.1.1)
+    http_parser.rb (0.6.0)
+    httparty (0.16.4)
+      mime-types (~> 3.0)
+      multi_xml (>= 0.5.2)
+    i18n (1.6.0)
+      concurrent-ruby (~> 1.0)
+    jaro_winkler (1.5.2)
+    loofah (2.2.3)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    method_source (0.9.2)
+    mime-types (3.2.2)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2018.0812)
+    mini_portile2 (2.4.0)
+    minitest (5.11.3)
+    multi_xml (0.6.0)
+    nokogiri (1.10.1)
+      mini_portile2 (~> 2.4.0)
+    parallel (1.14.0)
+    parser (2.6.0.0)
+      ast (~> 2.4.0)
+    powerpack (0.1.2)
+    proxy_fetcher (0.9.0)
+      http (~> 3.0)
+    psych (3.1.0)
+    public_suffix (3.0.3)
+    rack (2.0.6)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.0.4)
+      loofah (~> 2.2, >= 2.2.2)
+    railties (5.2.2)
+      actionpack (= 5.2.2)
+      activesupport (= 5.2.2)
+      method_source
+      rake (>= 0.8.7)
+      thor (>= 0.19.0, < 2.0)
+    rainbow (3.0.0)
+    rake (10.5.0)
+    retries (0.0.5)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    rubocop (0.65.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.5, != 2.5.1.1)
+      powerpack (~> 0.1)
+      psych (>= 3.1.0)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.4.0)
+    rubocop-rails_config (0.4.3)
+      railties (>= 3.0)
+      rubocop (~> 0.58)
+    ruby-progressbar (1.10.0)
+    thor (0.20.3)
+    thread_safe (0.3.6)
+    typhoeus (1.3.1)
+      ethon (>= 0.9.0)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.5)
+    unicode-display_width (1.4.1)
+    user-agent-randomizer (0.2.0)
+    zeitwerk (1.3.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  active_proxy!
+  activesupport (>= 5.0.0)
+  bundler (~> 1.16)
+  http (>= 3.0.0)
+  httparty (~> 0.16.4)
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  rubocop (~> 0.65.0)
+  rubocop-rails_config (~> 0.4.3)
+  typhoeus (~> 1.3.1)
+
+BUNDLED WITH
+   1.16.6

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 sebi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,39 @@
+# ActiveProxy
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/active_proxy`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'active_proxy'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install active_proxy
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/active_proxy.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/active_proxy.gemspec

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+lib = File.expand_path("lib", __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "active_proxy/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "active_proxy"
+  spec.version       = ActiveProxy::VERSION
+  spec.authors       = ["sebi"]
+  spec.email         = ["gore.sebyx@yahoo.com"]
+
+  spec.summary       = "Ruby proxy fetcher and manager"
+  spec.description   = "Easy to use ruby proxy fetcher, supports caching and retries"
+  spec.homepage      = "https://github.com/sebyx07/active_proxy"
+  spec.license       = "MIT"
+
+  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "nokogiri", ">= 1.6.8"
+  spec.add_dependency "proxy_fetcher", ">= 0.9.0"
+  spec.add_dependency "retries", ">= 0.0.5"
+  spec.add_dependency "user-agent-randomizer", ">= 0.2.0"
+  spec.add_dependency "zeitwerk", ">= 1.0.0"
+
+  spec.add_development_dependency "activesupport", ">= 5.0.0"
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "http", ">= 3.0.0"
+  spec.add_development_dependency "httparty", "~> 0.16.4"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rubocop", "~> 0.65.0"
+  spec.add_development_dependency "rubocop-rails_config", "~> 0.4.3"
+  spec.add_development_dependency "typhoeus", "~> 1.3.1"
+end

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'active_proxy'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/active_proxy/fetcher.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module ActiveProxy
+  class Fetcher
+    attr_reader :proxy_key, :options, :cache_store
+
+    def initialize(proxy_key, cache_store, options = {})
+      @proxy_key = proxy_key
+      @cache_store = cache_store
+      @options = options
+    end
+
+    def format_proxy_httparty
+      proxy = current_proxy
+      { http_proxyaddr: proxy[:address], http_proxyport: proxy[:port], timeout: 4 }
+    end
+
+    def format_proxy_full
+      proxy = current_proxy
+
+      "#{proxy[:type]}://#{proxy[:address]}:#{proxy[:port]}"
+    end
+
+    def format_proxy_typhoeus
+      { proxy: format_proxy_full }
+    end
+
+    def format_proxy_http
+      proxy = current_proxy
+      [proxy[:address], proxy[:port]]
+    end
+
+    def current_proxy
+      proxy = cache_store.fetch(cache_key("current")) do
+        _proxy = proxy_list.sample
+        log_message "Got new proxy #{_proxy}"
+        _proxy
+      end
+      return proxy unless proxy.nil?
+      cache_store.delete(cache_key("list"))
+      current_proxy
+    end
+
+    def user_agent
+      UserAgentRandomizer::UserAgent.fetch.string
+    end
+
+    private
+
+      def cache_key(item_name)
+        "HTTP-PROXY-#{proxy_key}-#{Process.pid}-#{item_name}"
+      end
+
+      def proxy_list
+        allowed_schemes = %w[http https]
+
+        cache_store.fetch(cache_key("list")) do
+          ProxyFetcher::Manager.new(proxy_manager_options).proxies.map do |proxy|
+            scheme = proxy.type.downcase
+            next unless allowed_schemes.include?(scheme)
+
+            { address: proxy.addr, port: proxy.port, type: proxy.type.downcase || "http" }
+          end.tap(&:compact!)
+        end
+      end
+
+      def call
+        handler = proc do |exception, attempt_number, total_delay|
+          log_message "Retry proxy, exception: #{exception.class}"
+          log_message exception.message[0..100]
+          log_message "retry attempt #{attempt_number}; #{total_delay.to_i} seconds have passed."
+          clear_current_proxy
+        end
+
+        with_retries(max_tries: max_retries_options, handler: handler) do
+          yield self
+        end
+      end
+
+      def max_retries_options
+        options[:max_retries] || 10
+      end
+
+      def proxy_manager_options
+        options[:proxy_manager_options] || { filters: { maxtime: "200" } }
+      end
+
+      def clear_current_proxy
+        new_list = proxy_list
+        new_list.delete(current_proxy)
+        cache_store.delete(cache_key("current"))
+        cache_store.write(cache_key("list"), new_list)
+      end
+
+      def log_message(msg)
+        return if ENV["LOG_ACTIVE_PROXY"].nil?
+        p msg
+      end
+  end
+end

data/lib/active_proxy/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ActiveProxy
+  VERSION = "1.0.0"
+end

data/lib/active_proxy.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "zeitwerk"
+require "retries"
+require "proxy_fetcher"
+require "user_agent_randomizer"
+
+Zeitwerk::Loader.for_gem.setup
+
+module ActiveProxy
+  class << self
+    def call(proxy_key, cache_store, options = {})
+      ActiveProxy::Fetcher.new(proxy_key, cache_store, options).send(:call) do |fetcher|
+        yield fetcher
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,255 @@
+--- !ruby/object:Gem::Specification
+name: active_proxy
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- sebi
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-03-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.8
+- !ruby/object:Gem::Dependency
+  name: proxy_fetcher
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+- !ruby/object:Gem::Dependency
+  name: retries
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+- !ruby/object:Gem::Dependency
+  name: user-agent-randomizer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.16.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.16.4
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.65.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.65.0
+- !ruby/object:Gem::Dependency
+  name: rubocop-rails_config
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.3
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+description: Easy to use ruby proxy fetcher, supports caching and retries
+email:
+- gore.sebyx@yahoo.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- active_proxy.gemspec
+- bin/console
+- bin/setup
+- lib/active_proxy.rb
+- lib/active_proxy/fetcher.rb
+- lib/active_proxy/version.rb
+homepage: https://github.com/sebyx07/active_proxy
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.8
+signing_key: 
+specification_version: 4
+summary: Ruby proxy fetcher and manager
+test_files: []