watobo 0.9.13 → 0.9.14

data/lib/watobo/interceptor/proxy.rb

@@ -19,7 +19,7 @@
 # along with WATOBO; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 # .
-# @private 
+# @private
 module Watobo#:nodoc: all
   module Interceptor
     #
@@ -31,17 +31,16 @@ module Watobo#:nodoc: all
 
       attr_accessor :proxy_mode
 
-      attr_accessor :contentLength
-      attr_accessor :contentTypes
+     # attr_accessor :contentLength
+      #attr_accessor :contentTypes
       attr_accessor :target
-      attr :www_auth
+      #attr :www_auth
       attr_accessor :client_certificates
-      
       def self.transparent?
         return true if ( Watobo::Conf::Interceptor.proxy_mode & Watobo::Interceptor::MODE_TRANSPARENT ) > 0
         return false
       end
-      
+
       def server
         @bind_addr
       end
@@ -91,7 +90,7 @@ module Watobo#:nodoc: all
           puts "[#{self.class}] stop"
           if @t_server.respond_to? :status
             puts @t_server.status
-          Thread.kill @t_server
+            Thread.kill @t_server
           @intercept_srv.close
           end
         rescue IOError => bang
@@ -111,6 +110,7 @@ module Watobo#:nodoc: all
       end
 
       def start()
+        @wait_queue = Queue.new
 
         if transparent?
           Watobo::Interceptor::Transparent.start
@@ -136,25 +136,30 @@ module Watobo#:nodoc: all
 
             c_sock = Watobo::HTTPSocket::ClientSocket.connect(session)
             Thread.exit if c_sock.nil?
+            # loop for reusing client connections
+            max_loop = 0
             loop do
             flags = []
             begin
-
-              request = c_sock.request
-
-              if request.nil? or request.empty? then
+              
+            # puts "#{c_sock} - read request"
+             request = c_sock.request
+             if request.nil? or request.empty? then
+                print "c/"
                 c_sock.close
                 Thread.exit
               end
-              puts "*[I] #{request.url}"
+              
+              url = ( request.url.to_s.length > 65 ) ? request.url.to_s.slice(0,65) + "..." : request.url.to_s
+              puts "\n[I] #{url}"
 
             rescue => bang
               puts "!!! Error reading client request "
               puts bang
               puts bang.backtrace
-              puts request.class
+             # puts request.class
              # puts request
-             closeSocket(c_sock)
+             c_sock.close
              Thread.exit
             #break
             end
@@ -164,18 +169,18 @@ module Watobo#:nodoc: all
               if request.first =~ /WATOBOPreview=([0-9a-zA-Z]*)/ then
 
                 puts "* preview requested ..."
-                puts request.url
+              #  puts request.url
 
                 hashid = $1
                 response = @preview[hashid]
 
                 if response then
                   c_sock.write response.join
-                  closeSocket(c_sock)
+                  c_sock.close
                 end
               end
             #next
-            Thread.current.exit
+            Thread.exit
             end
 
             request_intercepted = false
@@ -197,9 +202,9 @@ module Watobo#:nodoc: all
                   #notify(:modify_request, request, Thread.current)
                   Watobo.print_debug "send request to target"
                   @target.addRequest(request, Thread.current)
-                  puts "* stopping thread: #{Thread.current} ..."
+               #   puts "* stopping thread: #{Thread.current} ..."
                   Thread.stop
-                  puts "* released thread: #{Thread.current}"
+               #   puts "* released thread: #{Thread.current}"
                 else
                   p "! no target for editing request"
                 end
@@ -213,13 +218,15 @@ module Watobo#:nodoc: all
             begin
 
             s_sock, req, resp = @sender.sendHTTPRequest(request, :update_sids => true, :update_session => false, :update_contentlength => true, :www_auth => @www_auth, :client_certificates => @client_certificates)
+            
             if s_sock.nil? then
+              puts "s_sock is nil! bye, bye, ..."
               puts request if $DEBUG
               c_sock.write resp.join unless resp.nil?
               c_sock.close
             #Thread.kill Thread.current
             Thread.exit
-            next
+            #next
             end
 
             rescue => bang
@@ -232,16 +239,22 @@ module Watobo#:nodoc: all
 
             # check if response should be passed throug
             #Thread.current.exit if isPassThrough?(req, resp, s_sock, c_sock)
+            if isPassThrough?(req, resp, s_sock, c_sock)
+              Watobo::HTTPSocket.close s_sock
+              c_sock.close                
+              Thread.exit
+            end
             #p "no pass-through"
 
             begin
             # puts "* got response status: #{resp.status}"
               missing_credentials = false
               rs = resp.status
+               auth_type = AUTH_TYPE_NONE
               if rs =~ /^(401|407)/ then
                 missing_credentials = true
 
-                auth_type = AUTH_TYPE_NONE
+               
                 resp.each do |rl|
                   if rl =~ /^(Proxy|WWW)-Authenticate: Basic/i
                     auth_type = AUTH_TYPE_BASIC
@@ -272,11 +285,11 @@ module Watobo#:nodoc: all
                 resp.fix_content_length
 
                 end
-              else
+              end
+              unless auth_type == AUTH_TYPE_UNKNOWN or req.method =~ /^head/i
                 # don't try to read body if request method is HEAD
-                unless req.method =~ /^head/i
                 @sender.readHTTPBody(s_sock, resp, req, :update_sids => true)
-                 end
+                Watobo::HTTPSocket.close s_sock
               end
             rescue => bang
               puts "!!! could not send request !!!"
@@ -310,17 +323,29 @@ module Watobo#:nodoc: all
               end
 
              # puts ">> SEND TO CLIENT"
+         #    puts ">>C<< - Close: #{request.connection_close?}"
+             #request.headers("Connection"){ |h| puts h }
+             
              if missing_credentials
                resp.set_header("Connection", "close")
-                c_sock.write resp.join
-                c_sock.close
+             elsif request.connection_close? or resp.content_length < 0 or max_loop > 4
+               resp.set_header("Proxy-Connection","close")
+               resp.set_header("Connection","close")
              else
-              c_sock.write resp.join
-              end
-             # puts resp.join
-             # puts "-----"
-             # closeSocket(c_sock)
-
+               resp.set_header("Connection","keep-alive")
+               resp.set_header("Keep-Alive", "max=4, timeout=120")
+             end
+             
+             resp_data = resp.join
+             c_sock.write resp_data
+             c_sock.flush
+            # puts "= RESPONSE ="
+            # puts resp_data
+            # puts "---"
+            #  puts resp_data.unpack("H*")[0]
+            #  puts "==="
+            #  puts
+             
             rescue Errno::ECONNRESET
               print "x"
               #  puts "!!! ERROR (Reset): reading body"
@@ -337,11 +362,19 @@ module Watobo#:nodoc: all
             #return
             end
 
-            chat = Chat.new(request.copy, resp.copy, :source => CHAT_SOURCE_INTERCEPT)
-           # notify(:new_interception, chat)
+           chat = Chat.new(request.copy, resp.copy, :source => CHAT_SOURCE_INTERCEPT)
+
            Watobo::Chats.add chat
-           Thread.current.exit if missing_credentials
-end
+           
+           
+           if missing_credentials or request.connection_close? or resp.content_length < 0 or max_loop > 4
+             c_sock.close
+             Thread.exit
+           end
+           print "o"
+           max_loop += 1
+
+           end
           }
 
         end
@@ -400,24 +433,7 @@ end
           @preview = Hash.new
           @preview['ProxyTest'] = ["HTTP/1.0 200 OK\r\nServer: Watobo-Interceptor\r\nConnection: close\r\nContent-Type: text/html; charset=iso-8859-1\r\n\r\n<html><body>PROXY_OK</body></html>"]
 
-          # p @settings[:certificate_path]
-          # p @settings[:cert_file]
-          # p @settings[:key_file]
-          # crt_path = Watobo::Conf::Interceptor.certificate_path
-          # crt_file = Watobo::Conf::Interceptor.cert_file
-          # key_file = Watobo::Conf::Interceptor.key_file
-          # dh_key_file = Watobo::Conf::Interceptor.dh_key_file
-
-          # crt_filename = File.join(Watobo.base_directory, crt_path, crt_file)
-          # key_filename = File.join(Watobo.base_directory, crt_path, key_file)
-
-          #  @ctx = OpenSSL::SSL::SSLContext.new('SSLv23_server')
-          # @cert = OpenSSL::X509::Certificate.new(File.read(crt_filename))
-          # @key = OpenSSL::PKey::RSA.new(File.read(key_filename))
-
-          #@dh_key = OpenSSL::PKey::DH.new(File.read(dh_filename))
           @dh_key = Watobo::CA.dh_key
-          #  @ctx.ciphers = nil # ['TLSv1/SSLv3', 56, 56 ]
 
         rescue => bang
           puts "!!!could not read certificate files:"
@@ -548,7 +564,7 @@ end
           end
         end
         return false if match_ctype == false
-        puts "* pass ctype filter"
+        #puts "* pass ctype filter"
         match_rcode = true
         rcode_filter = @response_filter_settings[:response_code_filter]
         negate = @response_filter_settings[:negate_response_code_filter]
@@ -564,7 +580,7 @@ end
           end
         end
         return false if match_rcode == false
-        puts "* pass rcode filter"
+        #puts "* pass rcode filter"
         true
       end
 
@@ -574,10 +590,9 @@ end
       #
       #
       def pass_through(server, client, maxbytes = 0)
-        buf = ''
-        print "[~"
+       
         bytes_read = 0
-        while buf
+        while 1
           begin
           #timeout(2) do
             buf = nil
@@ -586,18 +601,25 @@ end
           rescue EOFError
           #client.write buf if buf
           #print "~]"
+         # msg = "\n[pass_through] EOF - "
+         # msg += buf.nil? ? "nil" : buf.size
+         #   puts msg
             return if buf.nil?
           rescue Errno::ECONNRESET
           # puts "!!! ERROR (Reset): reading body"
           # puts "* last data seen on socket: #{buf}"
-            print "R"
+         # msg = "!R - "
+         # msg += buf.nil? ? "nil" : buf.size
+         # msg << " !\n"
+         #   puts msg
+            
             return if buf.nil?
           rescue Timeout::Error
           #puts "!!! ERROR (Timeout): reading body"
           #puts "* last data seen on socket:"
           #client.write buf if buf
             print "T"
-            return if buf.nil?
+            return 
           rescue => bang
             puts "!!! could not read body !!!"
             puts bang
@@ -611,25 +633,24 @@ end
 
           begin
             return if buf.nil?
-            print "~"
+           # print "~"
             client.write buf
             bytes_read += buf.length
-            # puts "##{bytes_read} of #{maxbytes}"
+           # puts "#{server} #{bytes_read} of #{maxbytes}"
             if maxbytes > 0 and bytes_read >= maxbytes
-              print "~]"
-            return
+              #print "~]"
+               return
             end
-
           rescue Errno::ECONNRESET
-            print "~x]"
+            #print "~x]"
             #  puts "!!! ERROR (Reset): reading body"
             #  puts "* last data seen on socket: #{buf}"
             return
           rescue Errno::ECONNABORTED
-            print "~x]"
+           # print "~x]"
             return
           rescue Errno::EPIPE
-            print "~x]"
+           # print "~x]"
             return
           rescue => bang
             puts "!!! client communication broken !!!"
@@ -642,8 +663,7 @@ end
       end
 
       def transparent?
-        return true if ( @proxy_mode & Watobo::Interceptor::MODE_TRANSPARENT ) > 0
-        return false
+        ( @proxy_mode & Watobo::Interceptor::MODE_TRANSPARENT ) > 0
       end
 
       def read_request(socket)
@@ -809,6 +829,7 @@ end
 
       def isPassThrough?(request, response, s_sock, c_sock)
         begin
+         # return false if true
           reason = nil
           clen = response.content_length
           # no pass-through necessary if request method is HEAD
@@ -816,19 +837,25 @@ end
 
           if matchContentType?(response.content_type) then
             # first forward headers
-            c_sock.write response.join
+            #c_sock.write response.join
             reason = []
             reason.push "---> WATOBO: PASS_THROUGH <---"
             reason.push "Reason: Content-Type = #{response.content_type}"
           elsif clen > @contentLength
             # puts "PASS-THROUGH: #{response.content_length}"
-            c_sock.write response.join
+            #c_sock.write response.join
             reason = []
             reason.push "---> WATOBO: PASS_THROUGH <---"
             reason.push "Reason: Content-Length > #{@contentLength} (#{response.content_length})"
           end
 
           return false if reason.nil?
+         
+          response.remove_header("Keep-Alive")
+          response.set_header("Connection", "close")
+          
+          c_sock.write response.join
+          
           reason.push "* DO MANUAL REQUEST TO GET FULL RESPONSE *"
           response.push reason.join("\n")
           chat = Watobo::Chat.new(request, response, :source => CHAT_SOURCE_INTERCEPT)
@@ -837,45 +864,16 @@ end
 
           pass_through(s_sock, c_sock, clen)
           #  puts "* Close Server Socket..."
-          closeSocket(c_sock)
+          #closeSocket(c_sock)
           #  puts "* Close Client Socket..."
-          closeSocket(s_sock)
+          #closeSocket(s_sock)
           #  puts "... done."
-          true
-        rescue => bang
-          puts bang
-          puts bang.backtrace if $DEBUG
-        return false
-        end
-      end
-
-      def notify(event, *args)
-        if @event_dispatcher_listeners[event]
-          @event_dispatcher_listeners[event].each do |m|
-            m.call(*args) if m.respond_to? :call
-          end
-        end
-      end
-
-      def closeSocket(socket)
-        #puts socket.class
-        return false if socket.nil?
-        begin
-        #if socket.class.to_s =~ /SSLSocket/
-          if socket.respond_to? :sysclose
-          #socket.io.shutdown(2)
-          socket.sysclose
-          elsif socket.respond_to? :shutdown
-          socket.shutdown(2)
-          elsif socket.respond_to? :close
-          socket.close
-          end
           return true
         rescue => bang
           puts bang
           puts bang.backtrace if $DEBUG
         end
-        false
+        return false
       end
 
     end

data/lib/watobo/mixins/httpparser.rb

@@ -426,23 +426,7 @@ module Watobo#:nodoc: all
 
       end
 
-      #     def get_parm_names
-
-      #       parm_names=[]
-      #       parmlist=[]
-      #       parmlist.concat(get_parms)
-
-      #       parmlist.each do |p|
-      #        if p then
-      #           p.gsub!(/=.*/,'')
-      #           parm_names.push p
-      #         end
-      #       end
-
-      #       return parm_names
-
-      #     end
-
+    
       def header_value(header_name)
         header_values =[]
         self.headers.each do |header|
@@ -593,6 +577,20 @@ def content_encoding
       def has_body?
         self.body.nil? ? false : true
       end
+      
+      def __connection_close?
+        headers("Connection") do |h|
+          return true if h =~ /close/i
+        end
+        return false
+      end
+      
+      def connection_close?
+        headers("Connection") do |h|
+          return false if h =~ /keep\-alive/i
+        end
+        return true
+      end
 
       def has_header?(name)
         self.each do |l|
@@ -708,6 +706,7 @@ end
         rescue => bang
           puts "! no headers available !".upcase
           puts bang
+          puts bang.backtrace
           if $DEBUG
             puts bang.backtrace
             puts self.to_yaml

data/lib/watobo/mixins/shapers.rb

@@ -186,10 +186,12 @@ module Watobo#:nodoc: all
 
         def removeHeader(header)
           begin
-            eoh_index = self.length
-            eoh_index -= 1 unless self.body.nil?
-
-            self.delete_if {|x| self.index(x) <= eoh_index and x =~ /#{header}/i }
+            matches = []
+            headers.each_with_index do |h,i|
+              matches << i if h =~ /#{header}/i
+            end
+            matches.map{ |m| self.delete_at(m) }
+          
           rescue => bang
             puts bang
             puts bang.backtrace if $DEBUG

data/lib/watobo.rb

@@ -39,6 +39,7 @@ require 'net/ntlm'
 require 'drb'
 require 'nokogiri'
 require 'stringio'
+require 'mechanize'
 
 require 'watobo/constants'
 require 'watobo/utils'
@@ -56,10 +57,13 @@ require 'watobo/parser'
 require 'watobo/interceptor'
 require 'watobo/http_socket'
 
+# WORKAROUND FOR LINUX :(
+dont_know_why_REQUIRE_hangs = Mechanize.new
+
 # @private 
 module Watobo#:nodoc: all #:nodoc: all
 
-  VERSION = "0.9.13"
+  VERSION = "0.9.14"
 
   def self.base_directory
     @base_directory ||= ""

data/modules/passive/possible_login.rb

@@ -91,11 +91,8 @@ module Watobo#:nodoc: all
                     old_cookies = chat.request.cookies.select do |rc|
                       cookie_old = true
                       chat.response.new_cookies do |nc|
-                        if rc =~ /^#{nc.name}/
-                          rc_name, rc_value = rc.split("=") 
-                          cookie_old = false unless rc_value == nc.value
-                        end
-                         puts ":#{rc} - #{nc.name} - #{cookie_old}"                        
+                         cookie_old = false unless rc.value == nc.value
+                         #puts ":#{rc} - #{nc.name} - #{cookie_old}"                        
                       end
                       puts ":#{rc} >> #{cookie_old}"     
                       cookie_old

data/plugins/crawler/crawler.rb

@@ -19,62 +19,7 @@
 # along with WATOBO; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 # .
-begin
-  require 'mechanize'
-rescue LoadError
-  puts "To use the crawler plugin you must install the 'mechanize' gem."
-  puts "Simply enter the command 'gem install mechanize'"
-end
-
-
-
-
-path = File.expand_path(File.dirname(__FILE__))
-
 %w( constants bags grabber engine uri_mp).each do |l|
-  require File.join(path, "lib", l)
+  require_relative File.join("lib", l)
 end
  
-if $0 == __FILE__
-  # @private 
-module Watobo#:nodoc: all
-    module Conf
-      class Interceptor
-        def self.port
- #         8081
-        nil
-        end
-      end
-    end
-  end
-  
-  require 'yaml'
-  if ARGV.length > 0
-    url = ARGV[0]
-  end
- 
-  hook = lambda{ |agent, request|
-    begin
-      puts request.class
-      puts request.method
-      puts request.methods.sort
-     
-      exit
-      clean_jar = Mechanize::CookieJar.new
-      agent.cookie_jar.each{|cookie|
-        puts "Cookie: #{cookie.name}"
-        clean_jar.add! cookie unless cookie.name =~ /^box/i
-      }
-      exit unless agent.cookie_jar.empty?(request.url.to_s)
-      agent.cookie_jar = clean_jar
-    rescue => bang
-      puts bang
-      puts bang.backtrace
-    end
-
-  }
-  hook = nil
- 
-  crawler = Watobo::Crawler::Engine.new
-  crawler.run(url, :pre_connect_hook => hook, :num_grabbers => 1 )
-end

data/plugins/crawler/gui/crawler_gui.rb

@@ -266,15 +266,14 @@ module Watobo#:nodoc: all
           prefs.update general_settings
           prefs.update hook_settings
 
-          # puts prefs.to_yaml
+          add_update_timer(250)
 
-          Thread.new{
-            url = @url_txt.text
-            @crawler.run(url, prefs)
+          Thread.new(@url_txt.text,prefs){|turl,tprefs|
+            @crawler.run(turl, tprefs)
           }
 
           @start_button.text = 'Cancel'
-          add_update_timer()
+          
           
           rescue PasswordMatchError
             #puts "Passwords Don't Match!"