janis 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d364bdc4c0ee61a20d219b5ff7cca4c670d5b264
+  data.tar.gz: 274b8649f22c31649bfe00e4c4e49592feaf68bd
+SHA512:
+  metadata.gz: e458062c22c2ff58e28da48b7f86addd9482cadf41d274288d650ef0c639a9a7a84ca21c81441fdd0af9a643ea2a13051627c3aa9c3a587c334c031eda20ec9e
+  data.tar.gz: a7fb29a62a8fa47d7ef640e9aa2659dfc141dc461fe363b98d183da2036ffb59b3a2ed1be50666d2d90850cce6eb68d1a1246dad863db20258e5daddc57fcacc

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.0.0
+before_install: gem install bundler -v 1.10.5

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in janis.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Mariano Giagante
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -0,0 +1,93 @@
+# Janis
+
+Janis will help you find proxy servers quickly, by grabbing them from a list of many (hopefully available and up-to-date) proxy listing websites. You can also tell Janis to parse from a specific website and it will do it if it knows how to. If it doesn't you can improve it by adding new Parsers (more on this on Usage section).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'janis'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install janis
+
+## Usage
+From your own script/app or from irb, require the gem with:
+```ruby
+require 'janis'
+```
+And then do:
+```ruby
+Janis.find(max_amount_of_results)
+```
+
+That will gather proxy server info from all url's (and local files) included in the default source list, bringing a maximum of results specified in the argument.
+Note: Entries in the default source list can be disabled by commenting them out with a # at their beginning.
+
+## Extending Janis
+
+If there's a proxy listing website you consider reliable and up-to-date which you'd like to add it to the list: 
+
+1. Fork Janis repository.
+2. Define a module file following the format shown in /specific_parsers/template.rb. There, subclass ProxyWebsiteParser and override the #parse method.
+   Example:
+
+    ```ruby
+    class MyAwesomeProxyListParser < Janis::Parsing::WebSpecificParsers::ProxyWebsiteParser
+      
+      include CapybaraWithPanthomJs # optional - only if you use capybara-poltergeist for parsing
+      
+      def self.url
+        # url to the proxy list website you will be parsing in the #parse method
+      end
+      
+      def configure_capybara # optional - only if you use capybara-poltergeist for parsing
+        Capybara.configure { |c| c.app_host = url }
+      end
+      
+      def initialize
+        super
+        configure_capybara # optional - only if you use capybara-poltergeist for parsing
+        @session = new_session # optional - only if you use capybara-poltergeist for parsing
+        @session.visit(url) # optional - only if you use capybara-poltergeist for parsing
+        obtain_html_doc
+      end
+    
+      def parse
+          # Your code to parse the page's content and deliver an array of strings
+          # Those strings must have the format "IP:PORT_NUMBER"
+      end
+      
+      private
+      
+      def obtain_html_doc # optional - Redefine the way the html document to parse is obtained if you use capybara/poltergeist
+        @html_doc = Nokogiri.HTML(@session.html)
+      end
+      
+    end
+    ```
+    
+3. Implement #parse method so that it successfully returns an array of strings with the "IP:PORT_NUMBER" format.
+   Example output: ["1.1.1.1:3434", "2.2.2.2:3333", "255.3.1.4: 8787"]. The implementation must use Nokogiri, our parser dependency.
+
+4. Run the tests.
+5. If all tests pass, create a pull request.
+6. Wait for the applauses to come!
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/mgiagante/janis.
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+require "bundler/setup"
+require "janis"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+require "pry"
+Pry.start
+

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/janis.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'janis/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "janis"
+  spec.version       = Janis::VERSION
+  spec.authors       = ["Mariano Giagante"]
+  spec.email         = ["mariano.giagante@gmail.com"]
+  spec.license       = "MIT"
+  spec.summary       = %q{Janis grabs proxy servers from many websites for you!}
+  spec.description   = %q{It uses a source list with several testes websites to provide proxy servers to you. It can be extended with new ones as well.}
+  spec.homepage      = "http://www.github.com/mgiagante/janis"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 0"
+  spec.add_development_dependency "pry", "~> 0"
+  spec.add_runtime_dependency "nokogiri", "~> 0"
+  spec.add_runtime_dependency "poltergeist", "~> 0"
+end

data/lib/janis.rb

@@ -0,0 +1,36 @@
+require 'janis/version'
+require 'janis/parsing'
+require 'yaml'
+
+module Janis
+  
+  IP_PORT_SEPARATOR = ':'
+
+  def self.find(amount, path_to_list = './proxy_server_list.yml')
+
+		proxy_list = YAML.load_file("#{path_to_list}").split(' ')
+		results = []
+
+    proxy_list.each do |url| 
+      if results.size < amount
+        parsed_from_url = Parsing.parse(url) unless url.include?('#') # Elements should look like ["1.1.1.1:8080", "2.2.2.2:9090"]
+        results_from_this_url = parsed_from_url.map { |entry| convert_to_hash(entry) }
+        # Result should look like [ { ip: "1.1.1.1", port: "8080" }, { ip: "2.2.2.2", port: "9090" } ]
+        results += results_from_this_url
+      end
+		end
+
+    results[0..amount - 1]
+    
+	end
+  
+  private
+
+  def self.convert_to_hash(proxy_string)
+    { 
+      ip: proxy_string.split(IP_PORT_SEPARATOR).first,
+      port: proxy_string.split(IP_PORT_SEPARATOR).last 
+    }
+  end
+
+end

data/lib/janis/parser_factory.rb

@@ -0,0 +1,25 @@
+require 'janis/proxy_website_parser'
+Dir['./lib/janis/specific_parsers/*.rb'].each { |file| require file }
+
+module Janis    
+
+  module Parsing
+    
+    class ParserFactory
+      
+      def self.create_parser_for(url)
+        parsers = Janis::Parsing::SpecificParsers::ProxyWebsiteParser.subclasses
+        raise "No parsers available!" if parsers.empty?
+        parser_class = parsers.find do |klass| 
+          klass.url == url
+        end
+        raise "No parser available for #{url}" unless parser_class
+        parser_class.new
+      end
+
+		end   
+  
+  end
+
+end
+

data/lib/janis/parsing.rb

@@ -0,0 +1,14 @@
+require 'janis/validations'
+require 'janis/parser_factory'
+
+module Janis
+
+	module Parsing
+    
+    def self.parse(url)
+      ParserFactory.create_parser_for(url).parse
+    end
+
+	end
+
+end

data/lib/janis/proxy_website_parser.rb

@@ -0,0 +1,64 @@
+module Janis 
+
+  module Parsing
+
+    module SpecificParsers      
+      
+      class ProxyWebsiteParser
+        
+        attr_reader :url
+
+        @@subclasses = []
+
+        def self.subclasses
+          @@subclasses
+        end
+
+        def self.inherited(subclass)
+          add_subclass(subclass)
+        end
+
+        def initialize
+          @url = self.class.url
+        end
+
+        def self.url
+          raise "Subclass Responsibility!"
+        end
+
+        # It should return an array of strings. Each string should have the format "IP:PORT"
+        def parse
+          raise "Subclass Responsibility!"
+        end
+
+        private
+
+        def obtain_html_doc
+          if self.url.include?("http://")
+            Nokogiri::HTML(get_content_by_http(self.url))
+          elsif url.include?("file://")
+            Nokogiri::HTML(read_content_from_file(self.url))
+          else
+            raise "#{self.url} is not a supported URL!"
+          end  
+        end
+
+        def get_content_by_http(url)
+          open(self.url)
+        end
+
+        def read_content_from_file(url)
+          File::open(self.url.gsub('file://',''), 'r').read
+        end
+
+        def self.add_subclass(subclass)
+          @@subclasses << subclass
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/janis/specific_parsers/hide_my_ass.rb

@@ -0,0 +1,19 @@
+module Janis
+
+  module Parsing
+    
+    module SpecificParsers
+      class HideMyAssParser < ProxyWebsiteParser
+        def self.url
+          'http://proxylist.hidemyass.com'
+        end
+
+        def parse
+          # TODO: Implement parsing for hidemyass.com
+        end
+      end
+    end
+
+  end
+
+end

data/lib/janis/specific_parsers/parsing_tools/capybara_with_phantom_js.rb

@@ -0,0 +1,29 @@
+require 'capybara/poltergeist'
+require 'capybara/dsl'
+
+module CapybaraWithPhantomJs
+  include Capybara::DSL
+
+  # Create a new PhantomJS session in Capybara
+  def new_session
+
+    # Register PhantomJS (aka poltergeist) as the driver to use
+    Capybara.register_driver :poltergeist do |app|
+      Capybara::Poltergeist::Driver.new(app)
+    end
+
+    # Use XPath as the default selector for the find method
+    #Capybara.default_selector = :xpath
+
+    # Start up a new thread
+    @session = Capybara::Session.new(:poltergeist)
+
+    # Report using a particular user agent
+    @session.driver.headers = { 'User-Agent' =>
+      "Janis Parser ;)" }
+
+    # Return the driver's session
+    @session
+  end
+
+end

data/lib/janis/specific_parsers/proxy-list_org.rb

@@ -0,0 +1,72 @@
+require 'nokogiri';
+require './lib/janis/specific_parsers/parsing_tools/capybara_with_phantom_js'
+
+module Janis
+  
+  module Parsing
+
+    module SpecificParsers
+      class ProxyListOrgParser < ProxyWebsiteParser
+
+        include CapybaraWithPhantomJs
+
+        Struct.new('Row', :proxy, :country, :city, :type, :speed, :https_ssl)
+
+        def self.url
+          'http://proxy-list.org'
+        end
+
+        def initialize
+          super
+          configure_capybara
+          @session = new_session 
+          @session.visit(url)
+          obtain_html_doc
+        end
+
+        def configure_capybara
+          Capybara.configure { |c| c.app_host = url }
+        end
+
+        def parse
+          total_rows = []
+          total_rows += rows
+          [2,3,4,5,6,7,8,9,10].each do |page_number|
+            @session.click_link(page_number.to_s)
+            obtain_html_doc
+            total_rows += rows
+          end
+          total_rows
+
+          #TODO: This map is here to adapt #parse output to the one expected by Janis.find. Remove this when it starts accepting
+          #more info about each proxy server.
+          total_rows.map do |row|
+            row.proxy
+          end
+        end
+
+        private
+        
+        def obtain_html_doc
+          @html_doc = Nokogiri.HTML(@session.html)
+        end
+
+        def rows
+          rows_in_html = @html_doc.css('ul').select { |ul| ul.to_s.match /\d\d\d\./}
+          results = rows_in_html.map do |row_html| 
+            row_object = Struct::Row.new( #TODO: This should be an actual class, and should have methods to retrieve all attributes.
+            row_html.css('.proxy').children.last.text,
+            row_html.css('.country').text,
+            row_html.css('.city').text,
+            row_html.css('.type').text,
+            row_html.css('.speed').text,
+            row_html.css('.https').text
+                                        )
+          end
+        end
+      end
+    end
+
+  end
+
+end

data/lib/janis/specific_parsers/simple.rb

@@ -0,0 +1,23 @@
+module Janis
+
+  module Parsing
+
+    module SpecificParsers
+      class SimpleParser < ProxyWebsiteParser
+        PROXY_REGEX = /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\:\d{1,5}/
+        def initialize
+          super
+          @html_doc = obtain_html_doc
+        end        
+        def parse
+          @result ||= @html_doc.to_s.scan(PROXY_REGEX)
+        end
+        def self.url
+          'file://./test/html/simple.html'
+        end
+      end
+    end
+
+  end
+
+end

data/lib/janis/specific_parsers/template.rb

@@ -0,0 +1,36 @@
+=begin
+
+class MyAwesomeProxyListParser < Janis::Parsing::WebSpecificParsers::ProxyWebsiteParser
+
+  include CapybaraWithPanthomJs # optional - only if you use capybara-poltergeist for parsing
+
+  def self.url
+    # url to the proxy list website you will be parsing in the #parse method
+  end
+
+  def configure_capybara # optional - only if you use capybara-poltergeist for parsing
+    Capybara.configure { |c| c.app_host = url }
+  end
+
+  def initialize
+    super
+    configure_capybara # optional - only if you use capybara-poltergeist for parsing
+    @session = new_session # optional - only if you use capybara-poltergeist for parsing
+    @session.visit(url) # optional - only if you use capybara-poltergeist for parsing
+    obtain_html_doc
+  end
+
+  def parse
+      # Your code to parse the page's content and deliver an array of strings
+      # Those strings must have the format "IP:PORT_NUMBER"
+  end
+
+  private
+
+  def obtain_html_doc # optional - Redefine the way the html document to parse is obtained if you use capybara/poltergeist
+    @html_doc = Nokogiri.HTML(@session.html)
+  end
+
+end
+
+=end

data/lib/janis/validations.rb

@@ -0,0 +1,30 @@
+module Janis
+
+	module Parsing
+
+		module Validations
+
+      def self.validate(entry)
+        validate_matchable(entry)
+        validate_format(entry)
+        # Add specific validations like "must not include letters", "must not include special chars other than : or . . ."
+        # "numbers separated by the . must not have more than 3 digits"
+        # etcetera
+      end
+
+      private
+      
+      def self.validate_matchable(entry)
+        raise "Entry is does not respond to #match." unless entry.respond_to?(:match)
+      end
+      
+      def self.validate_format(entry)
+        format_regex = /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\:\d{1,5}/
+        raise "Entry has an invalid format." unless entry.matches? FORMAT_REGEX # This one covers unexpected situations 
+      end
+
+		end
+
+	end
+
+end

data/lib/janis/version.rb

@@ -0,0 +1,3 @@
+module Janis
+  VERSION = "0.1.0"
+end

data/proxy_server_list.yml

@@ -0,0 +1,5 @@
+http://proxy-list.org
+#http://incloak.es/proxy-list/
+#http://spys.ru/free-proxy-list/
+#http://www.samair.ru/proxy/
+#http://www.proxys.com.ar/

metadata

@@ -0,0 +1,151 @@
+--- !ruby/object:Gem::Specification
+name: janis
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Mariano Giagante
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-05-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: poltergeist
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: It uses a source list with several testes websites to provide proxy servers
+  to you. It can be extended with new ones as well.
+email:
+- mariano.giagante@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- janis.gemspec
+- lib/janis.rb
+- lib/janis/parser_factory.rb
+- lib/janis/parsing.rb
+- lib/janis/proxy_website_parser.rb
+- lib/janis/specific_parsers/hide_my_ass.rb
+- lib/janis/specific_parsers/parsing_tools/capybara_with_phantom_js.rb
+- lib/janis/specific_parsers/proxy-list_org.rb
+- lib/janis/specific_parsers/simple.rb
+- lib/janis/specific_parsers/template.rb
+- lib/janis/validations.rb
+- lib/janis/version.rb
+- proxy_server_list.yml
+homepage: http://www.github.com/mgiagante/janis
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Janis grabs proxy servers from many websites for you!
+test_files: []
+has_rdoc: