watobo 0.9.14 → 0.9.15

data/CHANGELOG.md

@@ -1,3 +1,32 @@
+Version 0.9.15
+==
+Fixes
+--
+**General**
+ * improved socket handling
+ * fixed some UTF-8 issues in passive modules
+ * added application/octet-stream to pass-through content-types
+ * `shapers.rb` didn't replace single quotes in `replace_post_parm`
+ * setting/replacing http-headers is now case-in-sensitive
+
+**Passive Modules**
+ * fixed `Disclosure_ipaddr`; now all IPs inside body are reported
+ 
+**Active Modules**
+ * Domino DB enumeration will now run on all requests
+ 
+**Crawler Plugin**
+ * extended allowed/excluded URL checks on full url path /w query
+
+News
+---
+ * Struts2 module for detecting CVE-2013-2251
+ * Struts2 module for detecting CVE-2013-1966
+ * conversation filter; added some shortcuts
+ * conversation table; added `send to->Crawler`
+ * client certificates; added PKCS12
+ * SAP passive module; extracts SAP headers, e.g. `sap-system`
+ 
 Version 0.9.14
 ===
 Fixes

data/config/interceptor.yml

@@ -5,6 +5,7 @@
   - application\/image
   - application\/pdf
   - application\/.*flash
+  - application\/octet-stream
   - image\/
   :content_length: 100000
 :port: 8081

data/lib/watobo/core/active_check.rb

@@ -92,10 +92,9 @@ module Watobo#:nodoc: all
         if id_string == '' then
         id_string = (Time.now.to_i + rand(10000)).to_s
         end
-        puts id_string
         #
         new_details[:fid] = Digest::MD5.hexdigest(id_string)
-        puts new_details[:fid]
+       
         puts new_details[:fid] if $DEBUG
 
         new_details[:module] = self.class.to_s

data/lib/watobo/core/client_cert_store.rb

@@ -0,0 +1,47 @@
+# .
+# client_cert_store.rb
+# 
+# Copyright 2013 by siberas, http://www.siberas.de
+# 
+# This file is part of WATOBO (Web Application Tool Box)
+#        http://watobo.sourceforge.com
+# 
+# WATOBO is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation version 2 of the License.
+# 
+# WATOBO is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with WATOBO; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+# .
+ @private 
+module Watobo#:nodoc: all
+  module ClientCertStore#:nodoc: all
+    @client_certs = {}
+    
+#    :ssl_client_cert
+#    :ssl_client_key
+#    :extra_chain_certs
+    
+    def self.clear
+      @client_certs.clear
+    end
+    
+    def self.set( site, cert )
+      return false if cert.nil?
+      @client_certs[ site.to_sym ] = cert
+      true
+    end
+    
+    def self.get( site )
+      return nil unless @client_certs.has_key? site.to_sym
+      @client_certs[ site.to_sym ]
+    end
+    
+end
+end

data/lib/watobo/core/scanner3.rb

@@ -19,10 +19,10 @@
 # along with WATOBO; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 # .
-# @private 
+# @private
 module Watobo#:nodoc: all
   class Scanner3
-    
+
     include Watobo::Constants
     include Watobo::Subscriber
 
@@ -34,55 +34,58 @@ module Watobo#:nodoc: all
     class Worker
       include Watobo::Constants
       include Watobo::Subscriber
-      
+
       attr :engine
-      
+
       STATE_IDLE = 0x00
       STATE_RUNNING = 0x01
       STATE_WAIT_FOR_LOGIN = 0x02
-      
       def state
         state = nil
-        @state_mutex.synchronize do 
+        @state_mutex.synchronize do
           state = @state
         end
         state
       end
-      
+
       def run
         @state_mutex.synchronize do @state = STATE_RUNNING; end
         Thread.new{ @engine.run }
       end
-      
+
       def start
-        @engine = Thread.new(@tasks, @logged_out_queue, @prefs){ |tasks, logged_out_queue, prefs|
+        @engine = Thread.new(@prefs){ |prefs|
           relogin_count = 0
           loop do
-            task = tasks.deq
+            Thread.current[:pos] = "wait for task"
+            task = @tasks.deq
             begin
               #puts "RUNNING #{task[:module]}"
               request, response = task[:check].call()
-              
-              
+
+              next if response.nil?
+
               unless prefs[:logout_signatures].empty? or prefs[:auto_login] == false
                 logged_out = false
                 prefs[:logout_signatures].each do |sig|
                   logged_out = true if response.join =~ /#{sig}/
                 end
-                
-                if logged_out 
+
+                if logged_out
+                  Thread.current[:pos] = "logged out"
                   @state_mutex.synchronize do @state = STATE_WAIT_FOR_LOGIN; end
-                  logged_out_queue.push self
+                  @logged_out_queue.push self
                   # stop current thread, will be waked-up by scanner
                   Thread.stop
                   relogin_count += 1
+                  Thread.current[:pos] = "set state"
                   @state_mutex.synchronize do @state = STATE_RUNNING; end
                   unless relogin_count > 5
                      request, response = task[:check].call()
-                  end 
+                  end
                 end
               end
-              
+
               unless prefs[:scanlog_name].nil?
                 chat = Chat.new(request, response, :id => 0, :chat_source => prefs[:chat_source])
                 Watobo::DataStore.add_scan_log(chat, prefs[:scanlog_name])
@@ -93,6 +96,7 @@ module Watobo#:nodoc: all
               puts bang.backtrace if $DEBUG
             ensure
               #puts "FINISHED #{task[:module]}"
+              Thread.current[:pos] = "scan_finished"
               notify(:task_finished, task[:module])
             end
             Thread.exit if relogin_count > 5
@@ -102,10 +106,21 @@ module Watobo#:nodoc: all
       end
 
       def stop
-        @state = STATE_IDLE
-        Thread.kill @engine if @engine.alive?
+        @state_mutex.synchronize{ @state = STATE_IDLE }
+        begin
+          return false if @engine.nil?
+          if @engine.alive?
+            puts "[#{self}] got stopped"
+            Thread.kill @engine
+          end
+          @engine = nil
+        rescue => bang
+          puts "!!! could not stop worker !!!"
+          puts bang
+          puts bang.backtrace
+        end
       end
-      
+
       def wait_for_login?
         state = false
         @state_mutex.synchronize do
@@ -113,7 +128,7 @@ module Watobo#:nodoc: all
         end
         state
       end
-      
+
       def running?
         @state_mutex.synchronize do
           running = ( @state == STATE_RUNNING )
@@ -129,7 +144,7 @@ module Watobo#:nodoc: all
         @relogin_count = 0
         @state_mutex = Mutex.new
         @state = STATE_IDLE
-      
+
       end
 
     end
@@ -139,42 +154,57 @@ module Watobo#:nodoc: all
     def tasks
       @tasks
     end
-    
+
     def status_running?
       ( status & SCANNER_RUNNING ) > 0
     end
-    
+
     def generation_finished?
       ( status & GENERATION_FINISHED ) > 0
     end
-    
+
     def finished?
+      # puts "num_waiting: #{@tasks.num_waiting}"
+      # puts "workers: #{@workers.length}"
+      # puts "generation finished? #{generation_finished?.class}"
+      # puts "num tasks: #{@tasks.size}"
+      # puts "running workers: #{running_workers}"
       return true if (
-         status_running? &&
-          ( @tasks.num_waiting == @workers.length ) &&
-          ( @tasks.size == 0 ) &&
-          generation_finished?
-         )
+      status_running? &&
+      ( @tasks.num_waiting == @workers.length ) &&
+      ( @tasks.size == 0 ) &&
+      generation_finished?
+      )
       false
     end
 
-    def running?()     
-        return false if (
-         status_running? &&
-          ( @tasks.num_waiting == @workers.length ) &&
-          ( @tasks.size == 0 ) &&
-          generation_finished?
-         )        
-        return true if status_running?
+    def running?()
+
+      return false if (
+      status_running? &&
+      ( @tasks.num_waiting == @workers.length ) &&
+      ( @tasks.size == 0 ) &&
+      generation_finished?
+      )
+      return true if status_running?
       return false
     end
 
     def stop()
+      print "\n[#{self}] stopping ... "
       begin
         @workers.each do |w|
           w.stop
         end
+        unless @ctrl_thread.nil?
+          if @ctrl_thread.alive?
+            puts "stop ctrl_thread"
+            Thread.kill @ctrl_thread
+          end
+        end
+        print "[OK]\n"
       rescue => bang
+        print "[OUTCH]\n"
         puts bang
         puts bang.backtrace if $DEBUG
       end
@@ -200,7 +230,7 @@ module Watobo#:nodoc: all
         YAML.load(YAML.dump(@task_counter))
       end
     end
-    
+
     def sum_total
       sum = 0
       @task_count_lock.synchronize do
@@ -208,7 +238,7 @@ module Watobo#:nodoc: all
       end
       sum
     end
-    
+
     def sum_progress
       sum = 0
       @task_count_lock.synchronize do
@@ -225,19 +255,28 @@ module Watobo#:nodoc: all
       @login_count = 0
       @max_login_count = 20
 
-      Thread.new{
+      @ctrl_thread = Thread.new{
         size = -1
         loop do
           if @tasks.num_waiting == @workers.length and @tasks.size == 0 and generation_finished?
-            @workers.map{|w| w.stop }
+            begin
+            puts "[#{self}] seems scan is finished. stopping workers now ..."
+            @workers.map{|w|
+              #puts "[]#{self}] stopping worker #{w}"
+              w.stop
+              }
             # suizide!
             Thread.exit
+            rescue => bang
+              puts bang
+              puts bang.backtrace
+            end
           end
-          
+
           if @logged_out.size == ( @workers.length - @tasks.num_waiting) or @tasks.num_waiting == @workers.size
             @logged_out.clear
             #puts "!LOGOUT DETECTED!\n#{@logged_out.size} - #{@workers.length} - #{@tasks.num_waiting}\n\n"
-            begin         
+            begin
               puts "Run login ..."
               login
               @workers.each do |wrkr|
@@ -246,14 +285,14 @@ module Watobo#:nodoc: all
                   wrkr.engine.run
                 end
               end
-                           
+
             rescue => bang
               puts bang
               puts bang.backtrace
             end
-          
+
           end
-          
+
           sleep 1
         end
       }
@@ -262,32 +301,32 @@ module Watobo#:nodoc: all
       msg = "\n[Scanner] Starting Scan ..."
       notify(:logger, LOG_INFO, msg )
       puts msg
-      
+
       # starting workers before check generation
       start_workers( @prefs)
       @max_tasks = 1000
-      
+
       # start check generation in seperate thread
       Thread.new{
         begin
-        set_status GENERATION_STARTED
-        @chat_list.uniq.each do |chat|
-        # puts chat.request.url.to_s
-          @active_checks.uniq.each do |ac|
-            ac.reset()
-            if site_alive?(chat) then
-              ac.generateChecks(chat){ |check|
-                while @tasks.size > @max_tasks
-                  sleep 1
-                end
-                task = { :module => ac,
-                  :check => check
+          set_status GENERATION_STARTED
+          @chat_list.uniq.each do |chat|
+          # puts chat.request.url.to_s
+            @active_checks.uniq.each do |ac|
+              ac.reset()
+              if site_alive?(chat) then
+                ac.generateChecks(chat){ |check|
+                  while @tasks.size > @max_tasks
+                    sleep 1
+                  end
+                  task = { :module => ac,
+                    :check => check
+                  }
+                  @tasks.push task
                 }
-                @tasks.push task
-              }
+              end
             end
           end
-        end
         rescue => bang
           puts bang
           puts bang.backtrace if $DEBUG
@@ -302,39 +341,36 @@ module Watobo#:nodoc: all
       @chat_list = chat_list
       @active_checks = []
       @passive_checks = passive_checks
-      
+
       @tasks = Queue.new
       @logged_out = Queue.new
-      
-      @workers = []
 
-     
+      @workers = []
 
       @status_lock = Mutex.new
 
       @task_count_lock = Mutex.new
       @task_counter = {}
+      
+      @ctrl_thread = nil
 
       # @onlineCheck = OnlineCheck.new(@project)
       msg = "Initializing Scanner ..."
       notify(:logger, LOG_INFO, msg)
       puts msg
-      
+
       @prefs = Watobo::Conf::Scanner.to_h
 
       @prefs.update prefs
-      #puts "set up scanner"
-      #puts @prefs[:login_chats]
-      #puts @prefs[:logout_signatures]
-      puts "= create scanner =" if $DEBUG
-      puts @prefs.to_yaml 
+
+      puts @prefs.to_yaml
 
       unique_checks = {}
       active_checks.each do  |x|
         if x.respond_to? :new
-          ac = x.new(self.object_id, @prefs)
+        ac = x.new(self.object_id, @prefs)
         else
-          ac = x
+        ac = x
         end
         unique_checks[ac.class.to_s] = ac unless unique_checks.has_key?(ac.class.to_s)
       end
@@ -348,7 +384,7 @@ module Watobo#:nodoc: all
 
         check.resetCounters()
         @chat_list.each_with_index do |chat, index|
-          #print "."
+        #print "."
           check.updateCounters(chat, @prefs)
           puts "* [#{index}] CheckCounter #{chat.id}: #{check.check_name} - #{check.numChecks}"
         end
@@ -361,7 +397,7 @@ module Watobo#:nodoc: all
           :progress => 0
         }
       end
-       @status = SCANNER_READY
+      @status = SCANNER_READY
       msg = "Scanner Ready!"
       notify(:logger, LOG_INFO, msg)
       puts msg
@@ -388,7 +424,7 @@ module Watobo#:nodoc: all
 
     def start_workers(check_prefs)
       num_workers = @prefs.has_key?(:max_parallel_checks) ? @prefs[:max_parallel_checks] : Watobo::Conf::Scanner.max_parallel_checks
-      
+
       puts "Starting #{num_workers} Workers ..."
 
       num_workers.times do |i|
@@ -401,26 +437,26 @@ module Watobo#:nodoc: all
             @task_counter[cn][:progress] += 1
           end
         }
-        
+
         @logout_count ||= 0
         @logout_count_lock ||= Mutex.new
         @num_waiting = 0
-          
+
         w.start
         @workers << w
       end
       print "\n"
 
     end
-    
+
     def login
-       puts "do relogin"
-       login_chats = Watobo::Conf::Scanner.login_chat_ids.uniq.map{|id| Watobo::Chats.get_by_id(id) }  
-           #  puts "running #{login_chats.length} login requests"
-           #  puts login_chats.first.class            
-             
+      puts "do relogin"
+      login_chats = Watobo::Conf::Scanner.login_chat_ids.uniq.map{|id| Watobo::Chats.get_by_id(id) }
+      #  puts "running #{login_chats.length} login requests"
+      #  puts login_chats.first.class
+
       @active_checks.first.runLogin(login_chats, @prefs)
-      
+
     end
 
     def site_alive?(chat)