proxy_daemon 0.1.3 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d2b9074ac6f2540105b251e49fb530bfedf864e7
-  data.tar.gz: 814414fde598324b3a3b2a06c2bcd88b2744a340
+  metadata.gz: 4b3331057afa6180824de6caef0c7850ce2ad626
+  data.tar.gz: 067c6efd89f48be51d25f8318ea66fe5c44a553f
 SHA512:
-  metadata.gz: 4d06f6fa08f208a4537406a82dd359a134b2a8822999636c5bb2e5a58a094f65a00df7135c4d57c162c320b00d909c5acd007f4235972c1bf59002c9e19a416c
-  data.tar.gz: 40c533a7e7930e23fd0b3bb14179bdc86704763878365441143b8062bac340e6ea00a491ee14ded9fb7abcc8c90c12b4afcadbf16db0b3368a377d3520c6a844
+  metadata.gz: 375d890481795fbeca35f98eaaf0dcfb1648a2fc21993a83be077aa195146881a16669d5845f62555ec0e0caf2772010e93b7944f7eba2135d67515433dbe9d2
+  data.tar.gz: 7e87ce478643d0261289738aeb021c03865012e28f3270c28e2b5379344ad5041c44ce5670ed9dbc3c410e35a32830d580116ce10d4710ddc73212dace5bc7c0

data/lib/proxy_daemon/daemon.rb

@@ -6,62 +6,79 @@ require 'colorize'
 module ProxyDaemon
   class Daemon
     attr_accessor :list
-    
-    def initialize(script, options)
-      @script = script
-      @proxies = ((options[:proxies].map { |proxy| proxy.gsub /\s+/, ' ' }) || []).shuffle
+
+    def initialize(script_or_parser, options = nil)
+      if script_or_parser.is_a? String then @script = script_or_parser
+      else
+        @script = '-'
+        
+        if script_or_parser.is_a? Hash then options = script_or_parser
+        else @parser = script_or_parser end
+      end
+      
+      @proxies = ((options[:proxies].map { |proxy| proxy.gsub /\s+/, ' ' }) || [])#.shuffle
       @urls = options[:urls] || []
-      @workers = [(options[:workers] || 10), @urls.count].min
+      @worker_processes = (options[:worker_processes] || 10)
       @tries = options[:tries] || 4
-    
+
       @threads = []
       @list = {}
       @semaphore = Mutex.new
     end
-  
-    def command(cmd, pipe, *params)
+
+    def command(cmd, pipe, params = {})
+      command = {command: cmd}.merge(params)
+      
       Thread.current[:command] = cmd
-      pipe.syswrite("#{cmd} #{params.join(' ')}\n")
+      pipe.puts(command.to_json)
     end
-  
+
     def listen(pipe, try = 0)
-      answer = (pipe.sysread(30) || '').strip
-  
+      packet = JSON.parse((pipe.gets || '').strip)
+      raise "Empty answer from worker process" if packet.empty?
+      answer = packet['answer']
+      raise "Process: '#{answer}'" if answer == 'timeout' || answer == 'error'
+      
       if answer == 'proxy'
         raise "Broken url: #{Thread.current[:url]}" if try > @tries
-      
+
         proxy = getProxy
+        log "Answer: #{answer}, data: #{packet['data']}".yellow, pipe
         log "Choosing new proxy: #{(proxy || 'nil').yellow}", pipe
-        command :proxy, pipe, proxy
-        command :url, pipe, Thread.current[:url]
+        command :proxy, pipe, proxy: proxy
+#        command :url, pipe, url: Thread.current[:url]
         listen pipe, try+1
-      elsif answer.empty?
-        raise "Empty answer from worker process"
-      elsif answer == 'timeout' || answer == 'error'
-        raise "Process: '#{answer}'"
-      elsif Thread.current[:command] == :url
-        if (buf = answer.match(/^set (.+?)[\s]*:[\s]*?(.+)$/i)); @list[buf[1]] = buf[2] end
-        log "Process: #{Thread.current[:url].cyan}: '#{answer.green}'", pipe
+      elsif answer == 'ok'
+        @semaphore.synchronize { @list.merge! packet['data'] } if packet.key? 'data'
+        log "Process: #{Thread.current[:url].cyan}: '#{answer.green}', data: #{packet['data'].to_json}", pipe
       else
-        log "Answer: #{answer}".green, pipe
+        log "Answer: #{answer}, data: #{packet['data'].to_json}".green, pipe
       end
     end
-  
+
     def worker
       IO.popen("#{@script}", 'r+') { |p|
         if (@script == '-' && p.nil?) # child process
-          if @block.nil?
-            $stderr.puts "[#{Process.pid}]".magenta + ' Empty block for parsing content'.red
+          if @block.nil? && @parser.nil?
+            $stderr.syswrite "[#{Process.pid}]".magenta + ' The parser is undefined and block wasn\'t given for parsing the content'.red + "\n"
+            $stderr.flush
+
             Kernel.exit!
           end
           
-          worker = ProxyDaemon::Worker.new
-          worker.call(&@block)
+          if @parser
+            worker = ProxyDaemon::Worker.new(@parser, @parse_method)
+            worker.call
+          elsif @block
+            worker = ProxyDaemon::Worker.new
+            worker.call(&@block)
+          end
         else
+          p.sync = true
           proxy = getProxy
           log "Starting loop with new proxy: ".green + "#{(proxy || 'nil').yellow}", p
-          command :proxy, p, proxy
-      
+          command :proxy, p, proxy: proxy
+
           begin
             loop do
               sleep(0.1)
@@ -74,41 +91,47 @@ module ProxyDaemon
               else
                 log 'Urls count: ' + "#{@urls.length}".green + ", #{url.green}"
                 Thread.current[:url] = url
-                command :url, p, url
+                command :url, p, url: url
                 listen p
               end
             end
-        
+
             log "Finishing loop".green, p
           rescue Exception => e
             @semaphore.synchronize {
-              log "Exception in main: " + "#{e.message.red}, '#{Thread.current[:url]}'".red, p
+              log "Exception in main:".red + " '#{Thread.current[:url]}'".yellow + " #{e.message.red}".red + "\n#{e.backtrace.join("\n").red}\n", p
               command :exit, p
               #puts e.backtrace
             }
           end
         end
       }
-    
+
       if @urls.length > 0
         # @threads << Thread.new(&(->{worker}))
         # @threads.last.join
         worker
       end
     end
-  
-    def start(&block)
+
+    def start(options = nil, &block)
       @block = block if block_given?
+      @parse_method = options[:parse] if options && options.key?(:parse)
+      worker_processes = [@worker_processes, @urls.count].min
       
       begin
-        puts "[main] Starting " + "#{@workers}".yellow + " workers:"
-        @workers.times { |i| @threads << Thread.new(&(->{worker})) }
+        puts "[main] Starting " + "#{worker_processes}".yellow + " workers:"
+        worker_processes.times { |i| @threads << Thread.new(&(->{worker})) }
         @threads.each { |t| t.join }
       rescue Interrupt => e
         puts "[main] Interrupted by user".yellow
       end
     end
-  
+
+    def add_urls(urls)
+      @semaphore.synchronize { @urls |= [*urls] }
+    end
+
   private
     def getProxy
       proxy = String.new
@@ -119,24 +142,25 @@ module ProxyDaemon
 
       proxy
     end
-  
+
     def getUrl
       url = String.new
       @semaphore.synchronize { url = @urls.shift }
-    
+
       url
     end
-  
+
     def finished(msg, pipe = 0)
       unless @finished
         @finished = true
         @semaphore.synchronize { log msg, pipe }
       end
     end
-  
+
     def log(msg, pipe = nil)
-      unless pipe.nil? then puts "[#{pipe.pid}]".blue + " #{msg}"
-      else puts msg end
+      unless pipe.nil? then $stdout.syswrite "[#{pipe.pid}]".blue + " #{msg}\n"
+      else $stdout.syswrite "#{msg}\n" end
+      $stdout.flush
     end
   end
-end
+end

data/lib/proxy_daemon/version.rb

@@ -1,3 +1,3 @@
 module ProxyDaemon
-  VERSION = '0.1.3'
+  VERSION = '0.1.5'
 end

data/lib/proxy_daemon/worker.rb

@@ -6,18 +6,21 @@ require 'openssl'
 module ProxyDaemon
   class Worker
     attr_accessor :url
-    
-    def initialize
+
+    def initialize(parser = nil, parse_method = nil)
       @client = Net::HTTP.Proxy(nil, nil)
+      @parser = parser
+      @parse_method = parse_method
       @url = ''
     end
-  
+
     def listen
       begin
         command = ''
         Timeout::timeout(6) {
           command = ($stdin.gets || String.new).strip
-          
+          log ('Task: ' + command.to_s.yellow)
+
           if command.empty?
             log "Got empty answer from daemon, exiting...".yellow
             raise Timeout::Error
@@ -27,13 +30,16 @@ module ProxyDaemon
         answer 'timeout'
         Kernel.exit!
       end
-  
+
       command
     end
-  
-    def answer(command)
+
+    def answer(answer, data = nil)
       begin
-        $stdout.puts "#{command}"
+        pack = {answer: answer}
+        pack[:data] = data if data
+        
+        $stdout.puts pack.to_json
         $stdout.flush
       rescue Errno::EPIPE => e
         log 'Broken pipe with daemon, exiting...'.yellow
@@ -42,7 +48,7 @@ module ProxyDaemon
         log e.inspect.red
       end
     end
-  
+
     def process(url)
       begin
         uri = URI(url)
@@ -63,31 +69,30 @@ module ProxyDaemon
             @page = http.request(req)
           }
         }
-      
+
         if (!@page.is_a?(Net::HTTPOK) || @page.body.empty?)
           log @page
           raise Net::HTTPBadResponse
         end
-      
+
         res = parse(@page.body)
-        
-        if !!res == res || res.nil?; answer(res ? 'ok' : 'error')
-        elsif res.is_a? Array; answer("set #{res[0]}:#{res[1]}")
-        else; answer('error') end
+
+        if !!res === res || res.nil? then answer(res ? 'ok' : 'error')
+        else  answer('ok', res) end
       rescue Timeout::Error, Errno::ETIMEDOUT, Errno::ECONNREFUSED,
       Errno::EINVAL, Errno::ECONNRESET, Errno::ENETUNREACH, SocketError, EOFError,
-      TypeError, Net::HTTPExceptions, Net::HTTPBadResponse, OpenSSL::SSL::SSLError => e
-        log "proxy".red + " in #{'process'.yellow}: #{e.inspect.red}"
+      TypeError, Zlib::BufError, Net::HTTPExceptions, Net::HTTPBadResponse, OpenSSL::SSL::SSLError => e
+        log 'proxy'.red + " in #{'process'.yellow}: #{e.inspect.red}"
         answer 'proxy'
       rescue Interrupt => e
         log 'Interrupted by user, exiting...'.yellow
         Kernel.exit!
       rescue Exception => e
-        log "rescue in #{'process'.yellow}: #{e.inspect},\n#{e.backtrace.reverse.join("\n").red}"
+        log "rescue in #{'process'.yellow}: #{e.inspect},\n#{e.backtrace.join("\n").red}"
         answer 'error'
       end
     end
-  
+
     def changeProxy(proxy)
       if proxy == 'localhost' || proxy.nil? || (proxy = proxy.split(/\s+/)).length < 2
         ENV['http_proxy'] = nil
@@ -95,42 +100,45 @@ module ProxyDaemon
         ENV['http_proxy'] = "http://#{proxy[0]}:#{proxy[1]}"
       end
     end
-  
+
     def parse(body)
-      raise NotImplementedError if @block.nil?
+      raise NotImplementedError if @parser.nil? && @block.nil?
       
-      @block.call(self, body)
+      @block.call(self, body) if @block
+      @parser.send(:"parse_#{@parse_method}", self, body) if @parser
 #      instance_exec body, &@block
     end
 
     def call(&block)
+      $stdout.sync = true
       @block = block if block_given?
       proxy = nil
 
       loop do
         begin
-          task = listen
-          case task
-          when /^proxy/
-            proxy = task.match(/^proxy\s*(.*)$/)[1]
+          task = JSON.parse(listen)
+          
+          case task['command']
+          when 'proxy'
+            proxy = task['proxy']
             changeProxy(proxy)
-          when /^url/
-            @url = task.match(/^url\s+(.+)$/)[1]
+            process(@url) unless @url.empty?
+          when 'url'
+            @url = task['url']
             process(@url)
-          when /^exit/
+          when 'exit'
             exit!
           end
-
-          #$stderr.puts "[child #{Process.pid}]".magenta + ' Task: ' + task.to_s.yellow
         rescue => e
           log "rescue in #{'call'.yellow}: #{e.inspect.red}"
           answer 'error'
         end
       end
     end
-  
+
     def log(msg)
-      $stderr.puts "[child #{Process.pid}]".magenta + " #{msg}"
+      $stderr.syswrite "[child #{Process.pid}]".magenta + " #{msg}\n"
+      $stderr.flush
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proxy_daemon
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - Michail Volkov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-11 00:00:00.000000000 Z
+date: 2016-07-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -95,7 +95,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.4
 signing_key: 
 specification_version: 4
 summary: Simple daemon for grabbing sites via proxy servers.