proxy_daemon 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 92ab7dba218ec3dc7b687a8539ccb1bc0639875a
+  data.tar.gz: fb9416fd34e1da07431a5b4060895ef72ed91a7b
+SHA512:
+  metadata.gz: 9b14a8a2843644dc54116a941e6cdbb56ff3cb0671652cbdb2d148bccb13492a3763f24adebb73e801e9cccc3d5846783cb84fe43a837ee1ea7bee6fc0be9f24
+  data.tar.gz: bde510d4d99e0d1e009f8fc376293d46e63c64aa9a0fce3117db2d8fed1a49854248fe18be4536d9458a02f6d628c0020631b5cb15b6c3598be5cafe4e9cf265

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/*.gem

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,49 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, and in the interest of
+fostering an open and welcoming community, we pledge to respect all people who
+contribute through reporting issues, posting feature requests, updating
+documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free
+experience for everyone, regardless of level of experience, gender, gender
+identity and expression, sexual orientation, disability, personal appearance,
+body size, race, ethnicity, age, religion, or nationality.
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery
+* Personal attacks
+* Trolling or insulting/derogatory comments
+* Public or private harassment
+* Publishing other's private information, such as physical or electronic
+  addresses, without explicit permission
+* Other unethical or unprofessional conduct
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+By adopting this Code of Conduct, project maintainers commit themselves to
+fairly and consistently applying these principles to every aspect of managing
+this project. Project maintainers who do not follow or enforce the Code of
+Conduct may be permanently removed from the project team.
+
+This code of conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting a project maintainer at xbiznet@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. Maintainers are
+obligated to maintain confidentiality with regard to the reporter of an
+incident.
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 1.3.0, available at
+[http://contributor-covenant.org/version/1/3/0/][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/3/0/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in proxy_daemon.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Michail
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,41 @@
+# ProxyDaemon
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/proxy_daemon`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'proxy_daemon'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install proxy_daemon
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/proxy_daemon. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "proxy_daemon"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/proxy_daemon/daemon.rb

@@ -0,0 +1,122 @@
+require 'thread'
+require 'timeout'
+require 'json'
+require 'colorize'
+
+module ProxyDaemon
+  class Daemon
+    def initialize(script, options)
+      @script = script
+      @proxies = ((options[:proxies].map { |proxy| proxy.gsub /\s+/, ' ' }) || []).shuffle
+      @urls = options[:urls] || []
+      @workers = options[:workers] || 10
+      @tries = options[:tries] || 4
+    
+      @threads = []
+      @semaphore = Mutex.new
+    end
+  
+    def command(cmd, pipe, *params)
+      Thread.current[:command] = cmd
+      pipe.syswrite("#{cmd} #{params.join(' ')}\n")
+    end
+  
+    def listen(pipe, try = 0)
+      answer = (pipe.sysread(30) || '').strip
+  
+      if answer == 'proxy'
+        raise "Broken url: #{Thread.current[:url]}" if try > @tries
+      
+        proxy = getProxy
+        log "Choosing new proxy: #{(proxy || 'nil').yellow}", pipe
+        command :proxy, pipe, proxy
+        command :url, pipe, Thread.current[:url]
+        listen pipe, try+1
+      elsif answer.empty?
+        raise "Empty answer from worker process"
+      elsif answer == 'timeout' || answer == 'error'
+        raise "Process: '#{answer}'"
+      elsif Thread.current[:command] == :url
+        log "Process: #{Thread.current[:url].cyan}: '#{answer.green}'", pipe
+      else
+        log "Answer: #{answer}".green, pipe
+      end
+    end
+  
+    def worker
+      IO.popen("#{@script}", 'r+') { |p|
+        proxy = getProxy
+        log "Starting loop with new proxy: ".green + "#{(proxy || 'nil').yellow}", p
+        command :proxy, p, proxy
+      
+        begin
+          loop do
+            sleep(0.1)
+            url = getUrl
+
+            if url.nil?
+              finished "Links are finished! exitting...".green, p
+              command :exit, p
+              break
+            else
+              log 'Urls count: ' + "#{@urls.length}".green + ", #{url.green}"
+              Thread.current[:url] = url
+              command :url, p, url
+              listen p
+            end
+          end
+        
+          log "Finishing loop".green, p
+        rescue Exception => e
+          @semaphore.synchronize {
+            log "Exception in main: " + "#{e.message.red}, '#{Thread.current[:url]}'".red, p
+            command :exit, p
+            #puts e.backtrace
+          }
+        end
+      }
+    
+      if @urls.length > 0
+        # @threads << Thread.new(&(->{worker}))
+        # @threads.last.join
+        worker
+      end
+    end
+  
+    def start
+      puts "[main] Starting " + "#{@workers}".yellow + " workers:"
+      @workers.times { |i| @threads << Thread.new(&(->{worker})) }
+      @threads.each { |t| t.join }
+    end
+  
+  private
+    def getProxy
+      proxy = String.new
+      @semaphore.synchronize {
+        proxy = @proxies.first
+        @proxies.rotate!
+      }
+
+      proxy
+    end
+  
+    def getUrl
+      url = String.new
+      @semaphore.synchronize { url = @urls.shift }
+    
+      url
+    end
+  
+    def finished(msg, pipe = 0)
+      unless @finished
+        @finished = true
+        @semaphore.synchronize { log msg, pipe }
+      end
+    end
+  
+    def log(msg, pipe = nil)
+      unless pipe.nil? then puts "[#{pipe.pid}]".blue + " #{msg}"
+      else puts msg end
+    end
+  end
+end

data/lib/proxy_daemon/version.rb

@@ -0,0 +1,3 @@
+module ProxyDaemon
+  VERSION = '0.1.1'
+end

data/lib/proxy_daemon/worker.rb

@@ -0,0 +1,127 @@
+require 'colorize'
+require 'timeout'
+require 'net/http'
+require 'openssl'
+
+module ProxyDaemon
+  class Worker
+    def initialize
+      @client = Net::HTTP.Proxy(nil, nil)
+      @url = ''
+    end
+  
+    def listen
+      begin
+        command = ''
+        Timeout::timeout(6) {
+          command = ($stdin.gets || String.new).strip
+          
+          if command.empty?
+            log "Got empty answer by daemon, exiting...".yellow
+            raise Timeout::Error
+          end
+        }
+      rescue Timeout::Error
+        answer 'timeout'
+        Kernel.exit!
+      end
+  
+      command
+    end
+  
+    def answer(command)
+      begin
+        $stdout.puts "#{command}"
+        $stdout.flush
+      rescue Errno::EPIPE => e
+        log 'Broken pipe with daemon, exiting...'.yellow
+        Kernel.exit!
+      rescue => e
+        log e.inspect.red
+      end
+    end
+  
+    def process(url)
+      begin
+        uri = URI(url)
+        Timeout::timeout(15) {
+          @client = Net::HTTP.new(uri.host, uri.port)
+          @client.use_ssl = (uri.scheme == 'https')
+          @client.verify_mode = OpenSSL::SSL::VERIFY_NONE if uri.scheme == 'https'
+          #@client.set_debug_output($stderr)
+
+          @client.start { |http|
+            req = Net::HTTP::Get.new(uri,
+              'Connection' => 'keep-alive',
+              'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+              'Upgrade-Insecure-Requests' => '1',
+              'User-Agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.13 Safari/537.36',
+              'Accept-Language' => 'en-US,en;q=0.8,ru;q=0.6'
+            )
+            @page = http.request(req)
+          }
+        }
+      
+        if (!@page.is_a?(Net::HTTPOK) || @page.body.empty?)
+          log @page
+          raise Net::HTTPBadResponse
+        end
+      
+        answer(parse(@page.body) ? 'ok' : 'error')
+      rescue Timeout::Error, Errno::ETIMEDOUT, Errno::ECONNREFUSED,
+      Errno::EINVAL, Errno::ECONNRESET, Errno::ENETUNREACH, SocketError, EOFError,
+      TypeError, Net::HTTPExceptions, Net::HTTPBadResponse, OpenSSL::SSL::SSLError => e
+        log "proxy".red + " in #{'process'.yellow}: #{e.inspect.red}"
+        answer 'proxy'
+      rescue Interrupt => e
+        log 'Interrupted by user, exiting...'.yellow
+        Kernel.exit!
+      rescue Exception => e
+        log "rescue in #{'process'.yellow}: #{e.inspect}, #{e.backtrace.reverse.join.red}"
+        answer 'error'
+      end
+    end
+  
+    def changeProxy(proxy)
+      if proxy == 'localhost' || proxy.nil? || (proxy = proxy.split(/\s+/)).length < 2
+        ENV['http_proxy'] = nil
+      else
+        ENV['http_proxy'] = "http://#{proxy[0]}:#{proxy[1]}"
+      end
+    end
+  
+    def parse(body)
+      raise NotImplementedError
+    end
+
+    def call
+      proxy = nil
+
+      loop do
+        begin
+          task = listen
+          case task
+          when /^proxy/
+            proxy = task.match(/^proxy\s*(.*)$/)[1]
+            changeProxy(proxy)
+          when /^url/
+            @url = task.match(/^url\s+(.+)$/)[1]
+            process(@url)
+          when /^exit/
+            exit!
+          end
+
+          #$stderr.puts "[child #{Process.pid}]".magenta + ' Task: ' + task.to_s.yellow
+        rescue => e
+          log "rescue in #{'call'.yellow}: #{e.inspect.red}"
+          answer 'error'
+        end
+      end
+    end
+  
+  private
+    def log(msg)
+      $stderr.puts "[child #{Process.pid}]".magenta + " #{msg}"
+    end
+  end
+end

data/lib/proxy_daemon.rb

@@ -0,0 +1,7 @@
+require 'proxy_daemon/version'
+require 'proxy_daemon/daemon'
+require 'proxy_daemon/worker'
+
+module ProxyDaemon
+  # Your code goes here...
+end

data/proxy_daemon.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'proxy_daemon/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'proxy_daemon'
+  spec.version       = ProxyDaemon::VERSION
+  spec.authors       = ['Michail Volkov']
+  spec.email         = ['xbiznet@gmail.com']
+
+  spec.summary       = %q{Simple daemon for grabbing sites via proxy servers.}
+  spec.description   = %q{Daemon and base worker for grabbing site pages via list of proxies in parallel.}
+  spec.homepage      = 'http://rubygems.org/gems/proxy-daemon'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = 'bin'
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.11'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  
+  spec.add_runtime_dependency 'colorize', '~> 0.7.7'
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification
+name: proxy_daemon
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Michail Volkov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-04-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.7
+description: Daemon and base worker for grabbing site pages via list of proxies in
+  parallel.
+email:
+- xbiznet@gmail.com
+executables:
+- console
+- setup
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/proxy_daemon.rb
+- lib/proxy_daemon/daemon.rb
+- lib/proxy_daemon/version.rb
+- lib/proxy_daemon/worker.rb
+- proxy_daemon.gemspec
+homepage: http://rubygems.org/gems/proxy-daemon
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Simple daemon for grabbing sites via proxy servers.
+test_files: []
+has_rdoc: