janis 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dd867caecf4745d2c6ebaacf88f2d08d230a5cae
-  data.tar.gz: 1cf3fbca12716b54a3eb2221046ec5deac906bdb
+  metadata.gz: d5f0f9b8e3e10fbb572bb3d39194c58edbeaea5b
+  data.tar.gz: eed32f6e8a3d1c1542286a024c26f9bd17a2e886
 SHA512:
-  metadata.gz: bef546eb4304e212253765902496d132bcea5baaa2777a737fe6fc75716de3a5af016877faac9c7b0801b9a263a04a0c2ce5eba73d3ef9a0dc33ef39346b1140
-  data.tar.gz: 5054cd249222acdbe2322cc8feeec1d1a6bcb90e1fe98abf83aa11c4a1949d27764cdcb7bdf7c5ae1d7e9d574b30e99c8c69ae9b07013c8a55aa5d7d2efcc124
+  metadata.gz: c32861860e3bccbaf2027dc633eb56fc5ea3b6923a56f87792ab6df020e3a8b685ddc2d755e13aebfc6a8bf911130ccf48303cedadb3ec89657833e6aa00c167
+  data.tar.gz: 084faba3c410b9793f0665b8ac1b780d704c599326fec582d4a42f9122e3745ee3f34dd60056dc94293acadaffda2d1a6826fa1cd21eb85ba9da58968f54c6e8

data/README.md

@@ -1,3 +1,8 @@
+#####Dependency issues are welcome to be reported in this repo at Issues section. Please include:
+    1. Your Operating System + architecture (Example: "Ubuntu 32 bits").
+    2. Full error backtrace.
+    3. Your ruby version (you can see it by typing "ruby -v" in your command prompt.
+
 # Janis
 
 Janis will help you find proxy servers quickly, by grabbing them from a list of many (hopefully available and up-to-date) proxy listing websites. You can also tell Janis to parse from a specific website and it will do it if it knows how to. If it doesn't you can improve it by adding new Parsers (more on this on Usage section).
@@ -17,6 +22,17 @@ And then execute:
 Or install it yourself as:
 
     $ gem install janis
+    
+Then download the latest version of PhantomJS from http://phantomjs.org/download.html, according
+to your platform.
+
+Place the PhantomJs executable somewhere in your PATH.
+
+On Unix, you can see your path from your shell by typing '$PATH'.
+Common folders to place phantomjs binary in are /usr/bin and usr/local/bin.
+
+On Windows, you can consult your PATH from your system settings in "Environment Variables" section.
+C:\windows\system32\ is a common location you can place phantomjs.exe in.
 
 ## Usage
 From your own script/app or from irb, require the gem with:
@@ -85,8 +101,6 @@ If there's a proxy listing website you consider reliable and up-to-date which yo
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
-
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/mgiagante/janis.

data/Rakefile

@@ -2,9 +2,9 @@ require "bundler/gem_tasks"
 require "rake/testtask"
 
 Rake::TestTask.new(:test) do |t|
-  t.libs << "test"
+  t.libs << "spec"
   t.libs << "lib"
-  t.test_files = FileList['test/**/*_test.rb']
+  t.test_files = FileList['spec/**/*_spec.rb']
 end
 
 task :default => :test

data/janis.gemspec

@@ -24,4 +24,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pry"
   spec.add_runtime_dependency "nokogiri", ">= 1.6"
   spec.add_runtime_dependency "poltergeist"
+  spec.add_runtime_dependency "net-ping"
 end

data/lib/janis.rb

@@ -1,37 +1,67 @@
 require 'janis/version'
 require 'janis/parsing'
-require 'yaml'
+require 'janis/testing'
+
+
+
+# TODO: Sites to be supported for scraping
+# http://incloak.es/proxy-list/
+# http://spys.ru/free-proxy-list/
+# http://www.samair.ru/proxy/
+# http://www.proxys.com.ar/
 
 module Janis
   
   IP_PORT_SEPARATOR = ':'
-  PROXY_LIST_PATH = File.dirname(__FILE__) + '/../proxy_server_list.yml'
   
-  def self.find(amount, path_to_list = PROXY_LIST_PATH)
+  def self.find(amount, opts = {})
 
-		proxy_list = YAML.load_file("#{path_to_list}").split(' ')
-		results = []
-
-    proxy_list.each do |url| 
-      if results.size < amount
-        parsed_from_url = Parsing.parse(url) unless url.include?('#') # Elements should look like ["1.1.1.1:8080", "2.2.2.2:9090"]
-        results_from_this_url = parsed_from_url.map { |entry| convert_to_hash(entry) }
-        # Result should look like [ { ip: "1.1.1.1", port: "8080" }, { ip: "2.2.2.2", port: "9090" } ]
-        results += results_from_this_url
+    # Makes sure opts[:websites] is a subset of the supported websites. Otherwise, it takes the whole list.
+    if opts[:websites]
+      opts[:websites].each do |website|
+        raise "#{website} is not supported!" unless Janis.supported_websites.include?(website)
       end
-		end
+      websites = opts[:websites]
+    else
+      websites = Janis.supported_websites
+    end  
 
-    results[0..amount - 1]
-    
+    total_results = []
+      
+    websites.each do |website| 
+      if total_results.size < amount
+        new_results = Parsing.parse_from(website).map { |entry| build_proxy_hash(entry, website) }
+        total_results += new_results
+      end
+    end
+    opts[:criteria] ? Janis::Testing.filter_results(criteria, total_results[0..amount - 1]) : total_results[0..amount -1]
 	end
-  
+ 
+  def self.supported_websites
+    Janis::Parsing::SpecificParsers::ProxyWebsiteParser.subclasses.map { |klass| self.website_name_for(klass.to_s)}
+  end
+
   private
 
-  def self.convert_to_hash(proxy_string)
+  def self.build_proxy_hash(proxy_string, website)
     { 
       ip: proxy_string.split(IP_PORT_SEPARATOR).first,
-      port: proxy_string.split(IP_PORT_SEPARATOR).last 
+      port: proxy_string.split(IP_PORT_SEPARATOR).last,
+      source: website
     }
   end
 
+  #TODO: This should be probably moved to a name helper module
+  def self.website_name_for(parser_klass_name)
+    parser_klass_name.gsub(/::/, '/').
+    gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+    gsub(/([a-z\d])([A-Z])/,'\1_\2').
+    tr("-", "_").
+    gsub("_Parser","").
+    split('/').
+    last.
+    downcase.to_sym
+    #TODO: converts a parser class name to a :symbol_in_snake_case website name
+  end
+
 end

data/lib/janis/parser_factory.rb

@@ -7,14 +7,23 @@ module Janis
     
     class ParserFactory
       
-      def self.create_parser_for(url)
-        parsers = Janis::Parsing::SpecificParsers::ProxyWebsiteParser.subclasses
-        raise "No parsers available!" if parsers.empty?
-        parser_class = parsers.find do |klass| 
-          klass.url == url
-        end
-        raise "No parser available for #{url}" unless parser_class
-        parser_class.new
+      attr_reader :parser_klasses
+
+      def initialize
+        @parser_klasses = Janis::Parsing::SpecificParsers::ProxyWebsiteParser.subclasses
+      end
+
+      def create_parser(website_name)
+        namespacing_prefix = "Janis::Parsing::SpecificParsers::"
+        @parser_klasses.find { |parser_klass| parser_klass.to_s == namespacing_prefix + parser_klass_name_for(website_name) }.new
+      end
+
+      private
+
+      #TODO: This should be probably moved to a name helper module
+      # website_name should be a :symbol_in_snake_lower_case. eg: :hide_my_ass will mean HideMyAssParser
+      def parser_klass_name_for(website_name)
+        website_name.to_s.split('_').map { |word| word.capitalize}.join + "Parser"  
       end
 
 		end   

data/lib/janis/parsing.rb

@@ -5,8 +5,8 @@ module Janis
 
 	module Parsing
     
-    def self.parse(url)
-      ParserFactory.create_parser_for(url).parse
+    def self.parse_from(website)
+      ParserFactory.new.create_parser(website).parse
     end
 
 	end

data/lib/janis/proxy_website_parser.rb

@@ -4,6 +4,18 @@ module Janis
 
     module SpecificParsers      
       
+      class Proxy
+        
+        def initialize(attribs = {})
+          @attribs = attribs
+        end
+
+        def method_missing?(message)
+          @attribs[message] || super 
+        end
+
+      end
+
       class ProxyWebsiteParser
         
         attr_reader :url

data/lib/janis/specific_parsers/proxy-list_org.rb

@@ -1,5 +1,5 @@
 require 'nokogiri';
-require './lib/janis/specific_parsers/parsing_tools/capybara_with_phantom_js'
+require_relative 'parsing_tools/capybara_with_phantom_js'
 
 module Janis
   

data/lib/janis/testing.rb

@@ -0,0 +1,21 @@
+require 'net/ping'
+
+module Janis
+
+  module Testing
+
+    def self.connectable?(proxy)
+      host, port = proxy.split(':')
+      return Net::Ping::TCP.new(host, port).ping
+    end
+
+    def self.filter_results(criteria = [], results)
+      criteria.each do |criterion| # A criterion is a method that returns true or false about a proxy, like #connectable?
+        results.select! { |proxy| Janis::Testing.send(criterion, "#{proxy[:ip]}:#{proxy[:port]}") }
+      end
+      results
+    end
+
+  end
+
+end

data/lib/janis/version.rb

@@ -1,3 +1,3 @@
 module Janis
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,97 +1,111 @@
 --- !ruby/object:Gem::Specification
 name: janis
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Mariano Giagante
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-05-15 00:00:00.000000000 Z
+date: 2016-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.10'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '5.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '5.4'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '1.6'
 - !ruby/object:Gem::Dependency
   name: poltergeist
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: net-ping
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: It uses a source list with several testes websites to provide proxy servers
@@ -102,8 +116,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- ".travis.yml"
+- .gitignore
+- .travis.yml
 - Gemfile
 - LICENSE
 - README.md
@@ -118,11 +132,10 @@ files:
 - lib/janis/specific_parsers/hide_my_ass.rb
 - lib/janis/specific_parsers/parsing_tools/capybara_with_phantom_js.rb
 - lib/janis/specific_parsers/proxy-list_org.rb
-- lib/janis/specific_parsers/simple.rb
 - lib/janis/specific_parsers/template.rb
+- lib/janis/testing.rb
 - lib/janis/validations.rb
 - lib/janis/version.rb
-- proxy_server_list.yml
 homepage: http://www.github.com/mgiagante/janis
 licenses:
 - MIT
@@ -133,12 +146,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
@@ -148,4 +161,3 @@ signing_key:
 specification_version: 4
 summary: Janis grabs proxy servers from many websites for you!
 test_files: []
-has_rdoc: 

data/lib/janis/specific_parsers/simple.rb

@@ -1,23 +0,0 @@
-module Janis
-
-  module Parsing
-
-    module SpecificParsers
-      class SimpleParser < ProxyWebsiteParser
-        PROXY_REGEX = /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\:\d{1,5}/
-        def initialize
-          super
-          @html_doc = obtain_html_doc
-        end        
-        def parse
-          @result ||= @html_doc.to_s.scan(PROXY_REGEX)
-        end
-        def self.url
-          'file://./test/html/simple.html'
-        end
-      end
-    end
-
-  end
-
-end

data/proxy_server_list.yml

@@ -1,5 +0,0 @@
-http://proxy-list.org
-#http://incloak.es/proxy-list/
-#http://spys.ru/free-proxy-list/
-#http://www.samair.ru/proxy/
-#http://www.proxys.com.ar/