proxy_fetcher 0.15.1 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 071dd5bfeaf01174bf803981de6147f999e75b99a2d03c32e034b72dd5dc7c82
-  data.tar.gz: 1675ce07491e0cbce401c2a113ee7eccdcba0c4cf42ba7e380ad63be988ec91a
+  metadata.gz: a35a9ffcd025603a3fe87c281a3d9d19fc68a7c88ae75adb6f9ef63b8b55c121
+  data.tar.gz: 2a9657da7a409c3ae13e5425da5623e351a04a75495546613473698004efe2a1
 SHA512:
-  metadata.gz: 42037b0b55109b535e25284815a096154c79bca2f58b107311f5c53da5b9ec0242ab39a6dd9bae48ef3e863c7f43251167180266578ac177887bd3a63215ef1e
-  data.tar.gz: 188dd298f8ea03e5985d89e97f99a12ef5d5384eae03a78f9b77e5172c28ae5990dec64795817d52ea6895395679a6c2f2deafafa3f8617154afb32fc06bc972
+  metadata.gz: 6b74bd11f91f0ef06bd754d048fd28705859f6c30b1f4a126a0dfbaa4b697d099efd4c26d4f6a98cbc3c47367baef3a3b5a0a80e099bc3fa5f1bfc3108b51def
+  data.tar.gz: 7b37eddbe892c7ce0f7343d231a18166213a4c9dd2bdd3dbe3720fe1f1c799474a7c2a312adc56134e587d58aebe34bb2b0dad2da51a21a355e846759c1e084b

data/.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+- package-ecosystem: bundler
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "03:00"
+  open-pull-requests-limit: 10

data/.github/workflows/ci.yml

@@ -0,0 +1,58 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    name: >-
+      Ruby ${{ matrix.ruby }} (${{ matrix.gemfile }})
+    env:
+      CI: true
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ endsWith(matrix.ruby, 'head') || matrix.ruby == 'debug' || matrix.experimental }}
+    if: |
+      !(   contains(github.event.pull_request.title,  '[ci skip]')
+        || contains(github.event.pull_request.title,  '[skip ci]'))
+    strategy:
+      fail-fast: true
+      matrix:
+        experimental: [false]
+        os: [ ubuntu-latest ]
+        ruby:
+          - 2.6
+          - 2.7
+          - '3.0'
+          - '3.1'
+        gemfile:
+          - gemfiles/oga.gemfile
+          - gemfiles/nokogiri.gemfile
+        include:
+          - ruby: head
+            os: ubuntu-latest
+            gemfile: gemfiles/nokogiri.gemfile
+            experimental: true
+          - ruby: head
+            os: ubuntu-latest
+            gemfile: gemfiles/oga.gemfile
+            experimental: true
+          - ruby: jruby
+            os: ubuntu-latest
+            gemfile: gemfiles/nokogiri.gemfile
+            experimental: true
+          - ruby: truffleruby
+            os: ubuntu-latest
+            gemfile: gemfiles/nokogiri.gemfile
+            experimental: true
+    steps:
+      - name: Repo checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - name: Run tests
+        timeout-minutes: 10
+        run: bundle exec rake spec

data/CHANGELOG.md

@@ -6,6 +6,15 @@ Reverse Chronological Order:
 
 ...
 
+## `0.17.0` (2023-06-02)
+
+* Remove dead providers
+
+## `0.16.0` (2022-02-04)
+
+* Fix providers
+* Improve HTTP.rb dependency
+
 ## `0.15.1` (2021-02-17)
 
 * Support for Ruby 3.0

data/Gemfile

@@ -9,7 +9,7 @@ gem "oga", "~> 3.2"
 gem "rubocop", "~> 1.0"
 
 group :test do
-  gem "coveralls", require: false
+  gem "coveralls_reborn", require: false
   # Until I find a way to introduce other MITM proxy
   gem "webrick", "1.4.2"
   gem "evil-proxy", "~> 0.2"

data/gemfiles/nokogiri.gemfile

@@ -7,7 +7,7 @@ gemspec path: "../"
 gem "nokogiri", "~> 1.8"
 
 group :test do
-  gem "coveralls", require: false
+  gem "coveralls_reborn", require: false
   gem "webrick"
   gem "evil-proxy", "~> 0.2"
 end

data/gemfiles/oga.gemfile

@@ -7,7 +7,7 @@ gemspec path: "../"
 gem "oga", "~> 3.0"
 
 group :test do
-  gem "coveralls", require: false
+  gem "coveralls_reborn", require: false
   gem "webrick"
   gem "evil-proxy", "~> 0.2"
 end

data/lib/proxy_fetcher/providers/free_proxy_list.rb

@@ -11,7 +11,7 @@ module ProxyFetcher
 
       # [NOTE] Doesn't support filtering
       def xpath
-        '//table[@id="proxylisttable"]/tbody/tr'
+        "//table[./thead/tr/th[contains(text(), 'IP')]]/tbody/tr"
       end
 
       # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>

data/lib/proxy_fetcher/providers/free_proxy_list_socks.rb

@@ -11,7 +11,7 @@ module ProxyFetcher
 
       # [NOTE] Doesn't support filtering
       def xpath
-        '//table[@id="proxylisttable"]/tbody/tr'
+        "//table[./thead/tr/th[contains(text(), 'IP')]]/tbody/tr"
       end
 
       # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>

data/lib/proxy_fetcher/providers/free_proxy_list_ssl.rb

@@ -11,7 +11,7 @@ module ProxyFetcher
 
       # [NOTE] Doesn't support filtering
       def xpath
-        '//table[@id="proxylisttable"]/tbody/tr'
+        "//table[./thead/tr/th[contains(text(), 'IP')]]/tbody/tr"
       end
 
       # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>

data/lib/proxy_fetcher/providers/free_proxy_list_us.rb

@@ -11,7 +11,7 @@ module ProxyFetcher
 
       # [NOTE] Doesn't support filtering
       def xpath
-        '//table[@id="proxylisttable"]/tbody/tr'
+        "//table[./thead/tr/th[contains(text(), 'IP')]]/tbody/tr"
       end
 
       # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>

data/lib/proxy_fetcher/version.rb

@@ -13,9 +13,9 @@ module ProxyFetcher
     # Major version number
     MAJOR = 0
     # Minor version number
-    MINOR = 15
+    MINOR = 17
     # Smallest version number
-    TINY  = 1
+    TINY  = 0
 
     # Full version number
     STRING = [MAJOR, MINOR, TINY].compact.join(".")

data/lib/proxy_fetcher.rb

@@ -41,9 +41,6 @@ module ProxyFetcher
     require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/mtpro"
     require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxy_list"
     require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxypedia"
-    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxyscrape_http"
-    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxyscrape_socks4"
-    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxyscrape_socks5"
     require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/xroxy"
   end
 

data/proxy_fetcher.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |gem|
   gem.license = "MIT"
   gem.required_ruby_version = ">= 2.3.0"
 
-  gem.add_runtime_dependency "http", ">= 3", "< 5"
+  gem.add_runtime_dependency "http", ">= 3"
 
   gem.add_development_dependency "rake", ">= 12.0"
   gem.add_development_dependency "rspec", "~> 3.9"

data/spec/proxy_fetcher/providers/proxy_classes_spec.rb

@@ -12,9 +12,6 @@ describe "Proxy classes" do
     [:mtpro,                 "MTPro"],
     [:proxy_list,            "ProxyList"],
     [:proxypedia,            "Proxypedia"],
-    [:proxyscrape_http,      "ProxyscrapeHTTP"],
-    [:proxyscrape_socks4,    "ProxyscrapeSOCKS4"],
-    [:proxyscrape_socks5,    "ProxyscrapeSOCKS5"],
     [:xroxy,                 "XRoxy"]
   ].each do |(provider_name, provider_klass)|
     describe Object.const_get("ProxyFetcher::Providers::#{provider_klass}") do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proxy_fetcher
 version: !ruby/object:Gem::Version
-  version: 0.15.1
+  version: 0.17.0
 platform: ruby
 authors:
 - Nikita Bulai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-17 00:00:00.000000000 Z
+date: 2023-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http
@@ -17,9 +17,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '3'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -27,9 +24,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '3'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '5'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -67,6 +61,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
@@ -101,9 +97,6 @@ files:
 - lib/proxy_fetcher/providers/mtpro.rb
 - lib/proxy_fetcher/providers/proxy_list.rb
 - lib/proxy_fetcher/providers/proxypedia.rb
-- lib/proxy_fetcher/providers/proxyscrape_http.rb
-- lib/proxy_fetcher/providers/proxyscrape_socks4.rb
-- lib/proxy_fetcher/providers/proxyscrape_socks5.rb
 - lib/proxy_fetcher/providers/xroxy.rb
 - lib/proxy_fetcher/proxy.rb
 - lib/proxy_fetcher/utils/http_client.rb
@@ -143,7 +136,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Ruby gem for dealing with proxy lists from different providers

data/lib/proxy_fetcher/providers/proxyscrape_http.rb

@@ -1,65 +0,0 @@
-# frozen_string_literal: true
-
-require "csv"
-
-module ProxyFetcher
-  module Providers
-    # FreeProxyList provider class.
-    class ProxyscrapeHTTP < Base
-      # Provider URL to fetch proxy list
-      def provider_url
-        "https://api.proxyscrape.com/v2/?request=getproxies&protocol=http"
-      end
-
-      # Loads provider HTML and parses it with internal document object.
-      #
-      # @param url [String]
-      #   URL to fetch
-      #
-      # @param filters [Hash]
-      #   filters for proxy provider
-      #
-      # @return [Array]
-      #   Collection of extracted proxies with ports
-      #
-      def load_document(url, filters = {})
-        html = load_html(url, filters)
-
-        CSV.parse(html, col_sep: "\t").map(&:first)
-      end
-
-      # Fetches HTML content by sending HTTP request to the provider URL and
-      # parses the txt document to return all the proxy entries (ip addresses
-      # and ports).
-      #
-      # @return [Array]
-      #   Collection of extracted proxies with ports
-      #
-      def load_proxy_list(filters = {})
-        load_document(provider_url, filters)
-      end
-
-      # Converts String to <code>ProxyFetcher::Proxy</code> object.
-      #
-      # @param node [String]
-      #   String
-      #
-      # @return [ProxyFetcher::Proxy]
-      #   Proxy object
-      #
-      def to_proxy(node)
-        addr, port = node.split(":")
-
-        ProxyFetcher::Proxy.new.tap do |proxy|
-          proxy.addr = addr
-          proxy.port = Integer(port)
-          proxy.country = "Unknown"
-          proxy.anonymity = "Unknown"
-          proxy.type = ProxyFetcher::Proxy::HTTP
-        end
-      end
-    end
-
-    ProxyFetcher::Configuration.register_provider(:proxyscrape_http, ProxyscrapeHTTP)
-  end
-end

data/lib/proxy_fetcher/providers/proxyscrape_socks4.rb

@@ -1,65 +0,0 @@
-# frozen_string_literal: true
-
-require "csv"
-
-module ProxyFetcher
-  module Providers
-    # FreeProxyList provider class.
-    class ProxyscrapeSOCKS4 < Base
-      # Provider URL to fetch proxy list
-      def provider_url
-        "https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks4"
-      end
-
-      # Loads provider HTML and parses it with internal document object.
-      #
-      # @param url [String]
-      #   URL to fetch
-      #
-      # @param filters [Hash]
-      #   filters for proxy provider
-      #
-      # @return [Array]
-      #   Collection of extracted proxies with ports
-      #
-      def load_document(url, filters = {})
-        html = load_html(url, filters)
-
-        CSV.parse(html, col_sep: "\t").map(&:first)
-      end
-
-      # Fetches HTML content by sending HTTP request to the provider URL and
-      # parses the txt document to return all the proxy entries (ip addresses
-      # and ports).
-      #
-      # @return [Array]
-      #   Collection of extracted proxies with ports
-      #
-      def load_proxy_list(filters = {})
-        load_document(provider_url, filters)
-      end
-
-      # Converts String to <code>ProxyFetcher::Proxy</code> object.
-      #
-      # @param node [String]
-      #   String
-      #
-      # @return [ProxyFetcher::Proxy]
-      #   Proxy object
-      #
-      def to_proxy(html_node)
-        addr, port = html_node.split(":")
-
-        ProxyFetcher::Proxy.new.tap do |proxy|
-          proxy.addr = addr
-          proxy.port = Integer(port)
-          proxy.country = "Unknown"
-          proxy.anonymity = "Unknown"
-          proxy.type = ProxyFetcher::Proxy::SOCKS4
-        end
-      end
-    end
-
-    ProxyFetcher::Configuration.register_provider(:proxyscrape_socks4, ProxyscrapeSOCKS4)
-  end
-end

data/lib/proxy_fetcher/providers/proxyscrape_socks5.rb

@@ -1,65 +0,0 @@
-# frozen_string_literal: true
-
-require "csv"
-
-module ProxyFetcher
-  module Providers
-    # FreeProxyList provider class.
-    class ProxyscrapeSOCKS5 < Base
-      # Provider URL to fetch proxy list
-      def provider_url
-        "https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks5"
-      end
-
-      # Loads provider HTML and parses it with internal document object.
-      #
-      # @param url [String]
-      #   URL to fetch
-      #
-      # @param filters [Hash]
-      #   filters for proxy provider
-      #
-      # @return [Array]
-      #   Collection of extracted proxies with ports
-      #
-      def load_document(url, filters = {})
-        html = load_html(url, filters)
-
-        CSV.parse(html, col_sep: "\t").map(&:first)
-      end
-
-      # Fetches HTML content by sending HTTP request to the provider URL and
-      # parses the txt document to return all the proxy entries (ip addresses
-      # and ports).
-      #
-      # @return [Array]
-      #   Collection of extracted proxies with ports
-      #
-      def load_proxy_list(filters = {})
-        load_document(provider_url, filters)
-      end
-
-      # Converts String to <code>ProxyFetcher::Proxy</code> object.
-      #
-      # @param node [String]
-      #   String
-      #
-      # @return [ProxyFetcher::Proxy]
-      #   Proxy object
-      #
-      def to_proxy(html_node)
-        addr, port = html_node.split(":")
-
-        ProxyFetcher::Proxy.new.tap do |proxy|
-          proxy.addr = addr
-          proxy.port = Integer(port)
-          proxy.country = "Unknown"
-          proxy.anonymity = "Unknown"
-          proxy.type = ProxyFetcher::Proxy::SOCKS5
-        end
-      end
-    end
-
-    ProxyFetcher::Configuration.register_provider(:proxyscrape_socks5, ProxyscrapeSOCKS5)
-  end
-end