watobo 0.9.12 → 0.9.13

data/lib/watobo/core/proxy.rb

@@ -1,7 +1,7 @@
 # .
 # proxy.rb
 # 
-# Copyright 2012 by siberas, http://www.siberas.de
+# Copyright 2013 by siberas, http://www.siberas.de
 # 
 # This file is part of WATOBO (Web Application Tool Box)
 #        http://watobo.sourceforge.com
@@ -19,7 +19,8 @@
 # along with WATOBO; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 # .
-module Watobo
+# @private 
+module Watobo#:nodoc: all
   class Proxy
       include Watobo::Constants
       
@@ -33,6 +34,10 @@ module Watobo
             super
           end
         end
+        
+      def to_yaml
+        @settings.to_yaml
+      end
 
 
       def has_login?

data/lib/watobo/core/request.rb

@@ -1,7 +1,7 @@
 # .
 # request.rb
 # 
-# Copyright 2012 by siberas, http://www.siberas.de
+# Copyright 2013 by siberas, http://www.siberas.de
 # 
 # This file is part of WATOBO (Web Application Tool Box)
 #        http://watobo.sourceforge.com
@@ -19,22 +19,107 @@
 # along with WATOBO; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 # .
-module Watobo
+# @private 
+module Watobo#:nodoc: all
   def self.create_request(url, prefs={})
-    u = "http://" unless url =~ /^http/
-    u << url
+    unless url =~ /^https?:\/\//
+      u = "http://#{url}"
+    else
+    u = url
+    end
+
     uri = URI.parse u
-     r = "GET #{uri.to_s} HTTP/1.0\r\n"
-     r << "Host: #{uri.host}" 
-     r.extend Watobo::Mixins::RequestParser
-     r.to_request
+    r = "GET #{uri.to_s} HTTP/1.0\r\n"
+    r << "Host: #{uri.host}\r\n"
+    r.extend Watobo::Mixins::RequestParser
+    r.to_request
   end
-  
-  module Request
+
+  class Request < Array
+    
+    attr :data
+    attr :url
+    attr :header
+    attr :cookies
+    
     def self.create request
       request.extend Watobo::Mixin::Parser::Url
       request.extend Watobo::Mixin::Parser::Web10
       request.extend Watobo::Mixin::Shaper::Web10
+     # request = Request.new(request)
+      
+    end
+    
+    def copy
+      c = YAML.load(YAML.dump(self))
+      Watobo::Request.new c
+    end
+    
+    def uniq_hash()
+      begin
+        settings = Watobo::Conf::Scanner.to_h
+        hashbase = site + method + path
+        
+        get_parm_names.sort.each do |p|
+          hashbase << p
+          hashbase << get_parm_value(p) if settings[:non_unique_parms].include?(p)
+        end
+
+        post_parm_names.sort.each do |p|
+        
+          hashbase << p
+          hashbase << post_parm_value(p) if settings[:non_unique_parms].include?(p)
+        end
+        # puts hashbase
+        return Digest::MD5.hexdigest(hashbase)
+      rescue => bang
+        puts bang
+        puts bang.backtrace if $DEBUG
+        return nil
+      end
+    end
+    
+    def parameters(*locations)
+      parms = []
+      parms.concat @data.parameters
+      parms.concat @url.parameters
+      parms
+    end
+    
+    def set(parm)
+      case parm.location
+      when :data
+        #
+       # replace_post_parm(parm.name, parm.value)
+       @data.set parm
+      when :url
+        @url.set parm
+      end
+      true
+    end
+
+    def initialize(r)
+      if r.respond_to? :concat
+        #puts "Create REQUEST from ARRAY"
+       self.concat r
+      elsif r.is_a? String
+        if r =~ /^http/
+          uri = URI.parse r
+          self << "GET #{uri.to_s} HTTP/1.0\r\n"
+          self << "Host: #{uri.host}\r\n"
+        else
+          r.extend Watobo::Mixins::RequestParser
+        self.concat r.to_request
+        end
+
+      end
+      self.extend Watobo::Mixin::Parser::Url
+      self.extend Watobo::Mixin::Parser::Web10
+      self.extend Watobo::Mixin::Shaper::Web10
+      
+      @url = Watobo::HTTP::Url.new(self)
+      @data = Watobo::HTTPData::WWW_Form.new(self)
+      @cookies = Watobo::HTTP::Cookies.new(self)
     end
   end
 end

data/lib/watobo/core/response.rb

@@ -1,7 +1,7 @@
 # .
 # response.rb
 # 
-# Copyright 2012 by siberas, http://www.siberas.de
+# Copyright 2013 by siberas, http://www.siberas.de
 # 
 # This file is part of WATOBO (Web Application Tool Box)
 #        http://watobo.sourceforge.com
@@ -19,12 +19,53 @@
 # along with WATOBO; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 # .
-module Watobo
-  module Response
+# @private 
+module Watobo#:nodoc: all
+  class Response < Array
+    def self.is_html?
+      
+    end
+    
+    def self.is_json?
+      
+    end
+    
+    
+    
+    
     def self.create( response )
+      raise ArgumentError, "Array Expected." unless response.is_a? Array
+      response.extend Watobo::Mixin::Parser::Url
       response.extend Watobo::Mixin::Parser::Web10
       response.extend Watobo::Mixin::Shaper::Web10
       response.extend Watobo::Mixin::Shaper::HttpResponse
     end
+    
+    def data
+      @data
+    end
+
+    def copy
+      c = YAML.load(YAML.dump(self))
+      Watobo::Request.new c
+    end
+    
+    def initialize(r)
+      if r.respond_to? :concat
+        #puts "Create REQUEST from ARRAY"
+       self.concat r
+      elsif r.is_a? String
+       raise ArgumentError, "Need Array"
+      end
+      self.extend Watobo::Mixin::Parser::Url
+      self.extend Watobo::Mixin::Parser::Web10
+      self.extend Watobo::Mixin::Shaper::Web10
+      self.extend Watobo::Mixin::Shaper::HttpResponse
+      
+      if content_type =~ /(html|text)/
+        self.extend Watobo::Parser::HTML
+      end
+      
+    end
   end
 end

data/lib/watobo/core/scanner.rb

@@ -1,7 +1,7 @@
 # .
 # scanner.rb
 # 
-# Copyright 2012 by siberas, http://www.siberas.de
+# Copyright 2013 by siberas, http://www.siberas.de
 # 
 # This file is part of WATOBO (Web Application Tool Box)
 #        http://watobo.sourceforge.com
@@ -19,7 +19,8 @@
 # along with WATOBO; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 # .
-module Watobo
+# @private 
+module Watobo#:nodoc: all
   class Scanner2
 
     attr :numTotalChecks
@@ -234,11 +235,9 @@ module Watobo
           @check_count += 1
           puts "CheckCount: #{@check_count}" if $DEBUG
           notify( :progress, m )
-          unless @prefs[:scanlog_name].nil?
-            if @prefs[:session_store].respond_to? :add_scan_log
+          unless @prefs[:scanlog_name].nil?            
               chat = Chat.new(request, response, :id => 0, :chat_source => @prefs[:chat_source])
-              @prefs[:session_store].add_scan_log(chat, @prefs[:scanlog_name])
-            end
+              Watobo::DataStore.add_scan_log(chat, @prefs[:scanlog_name])            
           end
         end
 
@@ -341,7 +340,7 @@ module Watobo
       puts @prefs.to_yaml if $DEBUG
 
       @filtered_chat_list = filteredChats(@chat_list, @prefs)
-      puts "#ActiveChecks: #{@active_checks.length}"
+      puts "#ActiveModules: #{@active_checks.length}"
 
       @active_checks.uniq.each do |m|
         puts m.class

data/lib/watobo/core/scanner3.rb

@@ -0,0 +1,439 @@
+# .
+# scanner3.rb
+# 
+# Copyright 2013 by siberas, http://www.siberas.de
+# 
+# This file is part of WATOBO (Web Application Tool Box)
+#        http://watobo.sourceforge.com
+# 
+# WATOBO is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation version 2 of the License.
+# 
+# WATOBO is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with WATOBO; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+# .
+# @private 
+module Watobo#:nodoc: all
+  class Scanner3
+    
+    include Watobo::Constants
+    include Watobo::Subscriber
+
+    SCANNER_READY = 0x0000
+    SCANNER_RUNNING = 0x0001
+
+    GENERATION_STARTED = 0x0100
+    GENERATION_FINISHED = 0x0200
+    class Worker
+      include Watobo::Constants
+      include Watobo::Subscriber
+      
+      attr :engine
+      
+      STATE_IDLE = 0x00
+      STATE_RUNNING = 0x01
+      STATE_WAIT_FOR_LOGIN = 0x02
+      
+      def state
+        state = nil
+        @state_mutex.synchronize do 
+          state = @state
+        end
+        state
+      end
+      
+      def run
+        @state_mutex.synchronize do @state = STATE_RUNNING; end
+        Thread.new{ @engine.run }
+      end
+      
+      def start
+        @engine = Thread.new(@tasks, @logged_out_queue, @prefs){ |tasks, logged_out_queue, prefs|
+          relogin_count = 0
+          loop do
+            task = tasks.deq
+            begin
+              #puts "RUNNING #{task[:module]}"
+              request, response = task[:check].call()
+              
+              
+              unless prefs[:logout_signatures].empty? or prefs[:auto_login] == false
+                logged_out = false
+                prefs[:logout_signatures].each do |sig|
+                  logged_out = true if response.join =~ /#{sig}/
+                end
+                
+                if logged_out 
+                  @state_mutex.synchronize do @state = STATE_WAIT_FOR_LOGIN; end
+                  logged_out_queue.push self
+                  # stop current thread, will be waked-up by scanner
+                  Thread.stop
+                  relogin_count += 1
+                  @state_mutex.synchronize do @state = STATE_RUNNING; end
+                  unless relogin_count > 5
+                     request, response = task[:check].call()
+                  end 
+                end
+              end
+              
+              unless prefs[:scanlog_name].nil?
+                chat = Chat.new(request, response, :id => 0, :chat_source => prefs[:chat_source])
+                Watobo::DataStore.add_scan_log(chat, prefs[:scanlog_name])
+              end
+            rescue => bang
+              puts "!!! #{task[:module]} !!!"
+              puts bang
+              puts bang.backtrace if $DEBUG
+            ensure
+              #puts "FINISHED #{task[:module]}"
+              notify(:task_finished, task[:module])
+            end
+            Thread.exit if relogin_count > 5
+            relogin_count = 0
+          end
+        }
+      end
+
+      def stop
+        @state = STATE_IDLE
+        Thread.kill @engine if @engine.alive?
+      end
+      
+      def wait_for_login?
+        state = false
+        @state_mutex.synchronize do
+          state = ( @state == STATE_WAIT_FOR_LOGIN )
+        end
+        state
+      end
+      
+      def running?
+        @state_mutex.synchronize do
+          running = ( @state == STATE_RUNNING )
+        end
+        running
+      end
+
+      def initialize(task_queue, logged_out_queue, prefs)
+        @engine = nil
+        @tasks = task_queue
+        @logged_out_queue = logged_out_queue
+        @prefs = prefs
+        @relogin_count = 0
+        @state_mutex = Mutex.new
+        @state = STATE_IDLE
+      
+      end
+
+    end
+    #
+    #  E N D   O F   W O R K E R
+
+    def tasks
+      @tasks
+    end
+    
+    def status_running?
+      ( status & SCANNER_RUNNING ) > 0
+    end
+    
+    def generation_finished?
+      ( status & GENERATION_FINISHED ) > 0
+    end
+    
+    def finished?
+      return true if (
+         status_running? &&
+          ( @tasks.num_waiting == @workers.length ) &&
+          ( @tasks.size == 0 ) &&
+          generation_finished?
+         )
+      false
+    end
+
+    def running?()     
+        return false if (
+         status_running? &&
+          ( @tasks.num_waiting == @workers.length ) &&
+          ( @tasks.size == 0 ) &&
+          generation_finished?
+         )        
+        return true if status_running?
+      return false
+    end
+
+    def stop()
+      begin
+        @workers.each do |w|
+          w.stop
+        end
+      rescue => bang
+        puts bang
+        puts bang.backtrace if $DEBUG
+      end
+    end
+
+    def cancel()
+      begin
+        @workers.each do |w|
+          w.stop
+        end
+      rescue => bang
+        puts bang
+        puts bang.backtrace if $DEBUG
+      end
+    end
+
+    def continue()
+      # TODO
+    end
+
+    def progress
+      @task_count_lock.synchronize do
+        YAML.load(YAML.dump(@task_counter))
+      end
+    end
+    
+    def sum_total
+      sum = 0
+      @task_count_lock.synchronize do
+        sum = @task_counter.values.inject(0){|i,v| i + v[:total] }
+      end
+      sum
+    end
+    
+    def sum_progress
+      sum = 0
+      @task_count_lock.synchronize do
+        sum = @task_counter.values.inject(0){|i,v| i + v[:progress] }
+      end
+      sum
+    end
+
+    def run( check_prefs={} )
+      # @sites_online.clear
+      @uniqueRequests = Hash.new
+      set_status_running
+
+      @login_count = 0
+      @max_login_count = 20
+
+      Thread.new{
+        size = -1
+        loop do
+          if @tasks.num_waiting == @workers.length and @tasks.size == 0 and generation_finished?
+            @workers.map{|w| w.stop }
+            # suizide!
+            Thread.exit
+          end
+          
+          if @logged_out.size == ( @workers.length - @tasks.num_waiting) or @tasks.num_waiting == @workers.size
+            @logged_out.clear
+            #puts "!LOGOUT DETECTED!\n#{@logged_out.size} - #{@workers.length} - #{@tasks.num_waiting}\n\n"
+            begin         
+              puts "Run login ..."
+              login
+              @workers.each do |wrkr|
+               # puts "State: #{wrkr.state}"
+                if wrkr.wait_for_login?
+                  wrkr.engine.run
+                end
+              end
+                           
+            rescue => bang
+              puts bang
+              puts bang.backtrace
+            end
+          
+          end
+          
+          sleep 1
+        end
+      }
+
+      @prefs.update check_prefs
+      msg = "\n[Scanner] Starting Scan ..."
+      notify(:logger, LOG_INFO, msg )
+      puts msg
+      
+      # starting workers before check generation
+      start_workers( @prefs)
+      @max_tasks = 1000
+      
+      # start check generation in seperate thread
+      Thread.new{
+        begin
+        set_status GENERATION_STARTED
+        @chat_list.uniq.each do |chat|
+        # puts chat.request.url.to_s
+          @active_checks.uniq.each do |ac|
+            ac.reset()
+            if site_alive?(chat) then
+              ac.generateChecks(chat){ |check|
+                while @tasks.size > @max_tasks
+                  sleep 1
+                end
+                task = { :module => ac,
+                  :check => check
+                }
+                @tasks.push task
+              }
+            end
+          end
+        end
+        rescue => bang
+          puts bang
+          puts bang.backtrace if $DEBUG
+        ensure
+          set_status GENERATION_FINISHED
+        end
+      }
+
+    end
+
+    def initialize(chat_list=[], active_checks=[], passive_checks=[], prefs={})
+      @chat_list = chat_list
+      @active_checks = []
+      @passive_checks = passive_checks
+      
+      @tasks = Queue.new
+      @logged_out = Queue.new
+      
+      @workers = []
+
+     
+
+      @status_lock = Mutex.new
+
+      @task_count_lock = Mutex.new
+      @task_counter = {}
+
+      # @onlineCheck = OnlineCheck.new(@project)
+      msg = "Initializing Scanner ..."
+      notify(:logger, LOG_INFO, msg)
+      puts msg
+      
+      @prefs = Watobo::Conf::Scanner.to_h
+
+      @prefs.update prefs
+      #puts "set up scanner"
+      #puts @prefs[:login_chats]
+      #puts @prefs[:logout_signatures]
+      puts "= create scanner =" if $DEBUG
+      puts @prefs.to_yaml 
+
+      unique_checks = {}
+      active_checks.each do  |x|
+        if x.respond_to? :new
+          ac = x.new(self.object_id, @prefs)
+        else
+          ac = x
+        end
+        unique_checks[ac.class.to_s] = ac unless unique_checks.has_key?(ac.class.to_s)
+      end
+      unique_checks.each_value do |check|
+        @active_checks << check
+      end
+
+      puts "#ActiveModules: #{@active_checks.length}"
+
+      @active_checks.uniq.each do |check|
+
+        check.resetCounters()
+        @chat_list.each_with_index do |chat, index|
+          #print "."
+          check.updateCounters(chat, @prefs)
+          puts "* [#{index}] CheckCounter #{chat.id}: #{check.check_name} - #{check.numChecks}"
+        end
+
+        # @numTotalChecks += check.numChecks
+        # cn = check.info[:check_name]
+        # puts "+ add check: #{cn}"
+        # notify(:logger, LOG_INFO, "add check #{cn}")
+        @task_counter[check.check_name] = { :total => check.numChecks,
+          :progress => 0
+        }
+      end
+       @status = SCANNER_READY
+      msg = "Scanner Ready!"
+      notify(:logger, LOG_INFO, msg)
+      puts msg
+    end
+
+    private
+
+    def set_status_running
+      s = ( status | SCANNER_RUNNING )
+      set_status( s )
+    end
+
+    def set_status(s)
+      @status_lock.synchronize {
+        @status |= s
+      }
+    end
+
+    def status
+      @status_lock.synchronize {
+        return @status
+      }
+    end
+
+    def start_workers(check_prefs)
+      num_workers = @prefs.has_key?(:max_parallel_checks) ? @prefs[:max_parallel_checks] : Watobo::Conf::Scanner.max_parallel_checks
+      
+      puts "Starting #{num_workers} Workers ..."
+
+      num_workers.times do |i|
+        print "... #{i + 1}"
+        w = Scanner3::Worker.new(@tasks, @logged_out, check_prefs)
+
+        w.subscribe(:task_finished){ |m|
+          @task_count_lock.synchronize do
+            cn = m.check_name
+            @task_counter[cn][:progress] += 1
+          end
+        }
+        
+        @logout_count ||= 0
+        @logout_count_lock ||= Mutex.new
+        @num_waiting = 0
+          
+        w.start
+        @workers << w
+      end
+      print "\n"
+
+    end
+    
+    def login
+       puts "do relogin"
+       login_chats = Watobo::Conf::Scanner.login_chat_ids.uniq.map{|id| Watobo::Chats.get_by_id(id) }  
+           #  puts "running #{login_chats.length} login requests"
+           #  puts login_chats.first.class            
+             
+      @active_checks.first.runLogin(login_chats, @prefs)
+      
+    end
+
+    def site_alive?(chat)
+      @sites_alive ||= Hash.new
+      site = chat.request.site
+      return true if @sites_alive.has_key? site
+
+      if Watobo::HTTPSocket.siteAlive?(chat)
+      @sites_alive[site] = true
+      return true
+      end
+      return false
+    end
+
+  end
+end