watobo 0.9.9.pre1 → 0.9.9.pre2

data/CHANGELOG

@@ -1,4 +1,4 @@
-= Version 0.9.9.pre1
+= Version 0.9.9
 == NEW
 * Time-based SQL injection module
 * new XSS module which gives a more accurate exploitability result

data/lib/watobo/core/interceptor.rb

@@ -136,8 +136,11 @@ module Watobo
               puts "*[I] #{request.url}"
 
             rescue => bang
-              puts "!!! Error reading client request "
-              puts bang
+             # puts "!!! Error reading client request "
+             # puts bang
+             # puts bang.backtrace
+             # puts request
+             # puts request
               Thread.current.exit
             #break
             end
@@ -586,6 +589,7 @@ module Watobo
       session = socket
       # puts "* read header ..."
       Watobo::HTTP.read_header(socket) do |line|
+       # puts line
         request.push line
       end
 

data/lib/watobo/gui/main_window.rb

@@ -270,8 +270,10 @@ module Watobo
       end
 
       def update_conversation_table(project)
+        return false if project.nil?
          @chatTable.showConversation(project.chats)
          @chatTable.apply_filter()  
+         return true
       end
       
       #

data/lib/watobo/gui/page_tree.rb

@@ -19,7 +19,7 @@
 # along with WATOBO; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 # .
-#require 'qcustomize.rb'
+ #require 'qcustomize.rb'
 
 module Watobo
   module Gui
@@ -200,8 +200,11 @@ module Watobo
         end
 
         form_name_item = nil
-        name = "#{form.name}"
-        form_name_item = self.appendItem(form_item, name, nil, nil) unless name.empty?
+        name = "undefined"
+        unless form.name.nil?
+        name = form.name unless form.name.empty?
+        end
+        form_name_item = self.appendItem(form_item, name, nil, nil) 
 
         unless form_name_item.nil?
           self.setItemData(form_name_item, form)
@@ -210,6 +213,10 @@ module Watobo
             self.setItemData(form_field_item, ff)
 
           end
+          form.buttons.each do |fb|
+            form_field_item = self.appendItem(form_name_item, fb.name, nil, nil)
+            self.setItemData(form_field_item, fb)
+          end
         end
 
         unless form_item.nil?

data/lib/watobo.rb

@@ -47,7 +47,7 @@ require 'watobo/framework'
 
 module Watobo
 
-  VERSION = "0.9.9.pre1"
+  VERSION = "0.9.9.pre2"
 
   def self.base_directory
     @base_directory ||= ""

data/plugins/crawler/gui/auth_frame.rb

@@ -34,7 +34,7 @@ module Watobo
 
           def update_fields
             self.getNumRows.times do |i|
-              self.getItemData(i,0).value = self.getItemText(i,1)
+              self.getItemData(i,0).value = self.getItemText(i,2)
             end
           end
 
@@ -61,8 +61,11 @@ module Watobo
             self.setItemData(lastRowIndex, 0, field)
             self.setItemText(lastRowIndex, 0, field.name)
             self.getItem(lastRowIndex, 0).justify = FXTableItem::LEFT
-            self.setItemText(lastRowIndex, 1, field.value)
+            self.setItemText(lastRowIndex, 1, field.type)
+            puts field.methods.sort
             self.getItem(lastRowIndex, 1).justify = FXTableItem::LEFT
+            self.setItemText(lastRowIndex, 2, field.value)
+            self.getItem(lastRowIndex, 2).justify = FXTableItem::LEFT
 
           end
           private
@@ -71,7 +74,7 @@ module Watobo
             begin
               row = item.row
               self.selectRow(row, false)
-              self.startInput(row,1)
+              self.startInput(row,2)
             rescue => bang
               puts bang
             end
@@ -87,14 +90,16 @@ module Watobo
 
           def initTable
             self.clearItems()
-            self.setTableSize(0, 2)
+            self.setTableSize(0, 3)
 
-            self.setColumnText( 0, "Field" )
-            self.setColumnText( 1, "Value" )
+            self.setColumnText( 0, "Name" )
+            self.setColumnText( 1, "Type" )
+            self.setColumnText( 2, "Value" )
 
             self.rowHeader.width = 0
             self.setColumnWidth(0, 100)
             self.setColumnWidth(1, 100)
+            self.setColumnWidth(2, 100)
           end
         end
 
@@ -205,23 +210,25 @@ module Watobo
 
                 page = nil
                 url = nil
+               
+               if @form_auth_url_txt.text.empty?
                 unless Watobo::Plugin::Crawler.start_url.nil?
                   uri = Watobo::Plugin::Crawler.start_url
+                end
                 else
-                  url = @form_auth_url_txt.text unless @form_auth_url_txt.text.empty?
+                   url = @form_auth_url_txt.text unless @form_auth_url_txt.text.empty?
                   uri = URI.parse(url) 
                 end
 
-                notify(:log, "GET PAGE << #{url}")
+                notify(:log, "GET PAGE << #{uri.to_s}")
                 @agent, page = @crawler.get_page(uri)
                 notify(:log, "PAGE LOADED")
                 @page_tree.page = page
               rescue => bang
-                notify(:log, "could not get page #{url}")
-                puts "Could not get page for #{url}"
-                puts url.class
+                notify(:log, "could not get page #{uri.to_s}")
+                puts "Could not get page for #{uri.to_s}"
                 puts bang
-                puts bang.backtrace
+                puts bang.backtrace if $DEBUG
               end
               true
             }
@@ -233,6 +240,9 @@ module Watobo
               form.fields.each do |field|
                 @form_fields_table.add_field field
               end
+              form.buttons.each do |field|
+                @form_fields_table.add_field field
+              end
             }
 
           end

data/plugins/crawler/gui/crawler_gui.rb

@@ -23,18 +23,23 @@ module Watobo
   module Plugin
     module Crawler
       def self.start_url
-        @start_url
+        @start_url ||= nil
       end
 
       def self.start_url=(url)
-        @start_url ||= ""
-        host = nil
+       # puts "Set Start-URL to #{url}"
+       # puts url.class
         begin
-          host = URI.parse(url).host
+        @start_url ||= nil
+        @start_url = url       
+        
+        @start_url = URI.parse(url) unless url.respond_to? :host
+          
         rescue => bang
-          host = nil
+          puts bang
+          @start_url = nil
         end
-        @start_url = url unless host.nil?
+       
         @start_url
       end
 
@@ -82,16 +87,19 @@ module Watobo
           @start_button.disable
 
           @url_txt.connect(SEL_COMMAND){ |sender, sel, item|
-            @start_button.setFocus() if url_valid?
+            if url_valid?
+           # @start_button.setFocus() 
+            Watobo::Plugin::Crawler.start_url = start_url
+            end
           }
 
           @url_txt.connect(SEL_CHANGED){
             if url_valid?
-              Watobo::Plugin::Crawler.start_url = start_url
+              
             @start_button.enable
             else
               @start_button.disable
-              Watobo::Plugin::Crawler.start_url = nil
+             # Watobo::Plugin::Crawler.start_url = nil
             end
           }
 
@@ -178,14 +186,21 @@ module Watobo
           auth = @settings_tabbook.auth.to_h
           # puts auth.to_yaml
           case auth[:auth_type]
+          when :basic
+            auth[:auth_uri] = start_url
           when :form
             if auth.has_key? :form
-              #puts ""
-              #@crawler.send_form(form)
-              if auth[:form].respond_to? :submit
+              begin
+              if auth[:form].buttons.length > 0
+                auth[:form].click_button 
+              #@crawler.send_form(form).
+              elsif auth[:form].respond_to? :submit
                 puts "Submitting Form"
                 p = auth[:form].submit()
-                puts p.class
+                #puts p.class
+              end
+              rescue => bang
+                puts bang
               end
             end
           end
@@ -214,8 +229,8 @@ module Watobo
 
         def url_valid?
           begin
-            return false unless @url_txt.text =~ /^https?:\/\//
-            url = URI.parse(@url_txt.text)
+            return false unless @url_txt.text.strip =~ /^https?:\/\//
+            url = URI.parse(@url_txt.text.strip)
             # puts url.host.class
             return true unless url.host.nil?
             return false

data/plugins/crawler/lib/engine.rb

@@ -22,25 +22,38 @@
 module Watobo
   module Crawler
     
-     #CRAWL_NONE = 0x00
-     # CRAWL_RUNNING = 0x01
-     # CRAWL_PAUSED = 0x02
-   
    class Agent < Mechanize
      
      def initialize(opts)
        super()
 
-        if ( opts.has_key? :proxy_host ) && ( opts.has_key? :proxy_port )
-            self.set_proxy( opts[:proxy_host], opts[:proxy_port] )
-        end
+        
         self.verify_mode = OpenSSL::SSL::VERIFY_NONE
         self.ignore_bad_chunking = true
+        self.keep_alive = false
+        
         self.user_agent = opts[:user_agent] if opts.has_key?(:user_agent)
+        
+        if opts.has_key? :username and opts.has_key? :password
+          unless opts[:username].empty? and opts[:password].empty?
+            
+             user = opts[:username]
+             pw = opts[:password]
+             uri = opts[:auth_uri]
+            #puts "Got Credentials for #{uri}: #{user} / #{pw}" 
+            self.add_auth(uri, user , pw )
+            # TODO: remove this workaround for a Mechanize Bug (#243)
+            p = self.get uri
+          end
+        end
+        
+        if ( opts.has_key? :proxy_host ) && ( opts.has_key? :proxy_port )
+            self.set_proxy( opts[:proxy_host], opts[:proxy_port] )
+        end
 
         if opts.has_key? :pre_connect_hook
            self.pre_connect_hooks << opts[:pre_connect_hook] if opts[:pre_connect_hook].respond_to? :call
-        end
+        end       
         
         unless opts[:cookie_jar].nil?
           clean_jar = Mechanize::CookieJar.new
@@ -52,21 +65,6 @@ module Watobo
         
      end
      
-     def initialize_UNUSED(opts)
-       super { |a|
-
-        if ( opts.has_key? :proxy_host ) && ( opts.has_key? :proxy_port )
-            a.set_proxy( opts[:proxy_host], opts[:proxy_port] )
-        end
-        a.verify_mode = OpenSSL::SSL::VERIFY_NONE
-        a.ignore_bad_chunking = true
-        a.user_agent = opts[:user_agent] if opts.has_key?(:user_agent)
-
-        if opts.has_key? :pre_connect_hook
-           a.pre_connect_hooks << opts[:pre_connect_hook] if opts[:pre_connect_hook].respond_to? :call
-        end
-        }
-     end
    end
       
     class Engine
@@ -107,6 +105,8 @@ module Watobo
     
       def initialize(opts={})
         @event_dispatcher_listeners = Hash.new
+        @status_lock = Mutex.new
+        
         @opts = {
           :submit_forms => true,
           :max_depth => 5,
@@ -125,6 +125,7 @@ module Watobo
           :root_path => "",  # regex
           :username => "",
           :password => "",
+          :auth_uri => nil,
           :auth_domain => "", # for ntlm auth
           :cookie_jar => nil
         }
@@ -149,25 +150,33 @@ module Watobo
       end
 
       def pause
-
+false
       end
 
       def cancel
-        @link_queue.clear
-        @page_queue.clear
+        puts "[CRAWLER] - CANCEL!!"
+       @status_lock.synchronize do
+         @engine_status = CRAWL_NONE
+       end
         @grabber_threads.each do |gt|
+          puts "Killing Thread #{gt}"
           gt.kill
+          gt.raise "CANCEL"
         end
         @grabber_threads.each{|t| t.join }
+        
+         @link_queue.clear
+        @page_queue.clear
         @grabber_threads.clear
          @link_keys.clear
         @link_counts.clear
         
         @form_keys.clear
         @form_counts.clear
-         @engine_status = CRAWL_NONE
-          notify( :update_status, current_status )
          
+          notify( :update_status, current_status )
+         puts "CANCELED - CANCELED"
+        # exit
       end
 
       def run(url, opts={})
@@ -192,19 +201,23 @@ module Watobo
 
         @link_queue.enq LinkBag.new(start_link, 0)
 
-puts "[Crawler] engine starting ..."
-#puts @opts.to_yaml
- notify(:log, "Crawling started ..." )
+puts "[Crawler] Engine Settings:"
+@opts.each do |k,v|  
+    puts "#{k}: #{v}"
+end
+puts "---"
+ notify(:log, "Crawling #{url} started ..." )
 
         @opts[:max_threads].times do |i|
           g = Grabber.new(@link_queue, @page_queue, @opts )
           @grabber_threads << g.run
         end
 
-        loop do
-          @engine_status = CRAWL_RUNNING
+        @engine_status = CRAWL_RUNNING
 
+        loop do
           pagebag = @page_queue.deq
+         
           process_links(pagebag)
           
           process_forms(pagebag)
@@ -212,14 +225,17 @@ puts "[Crawler] engine starting ..."
           
           puts "Links/Pages: #{@link_queue.size}/#{@page_queue.size}"
           notify( :update_status, current_status )
-          if @link_queue.empty? and @page_queue.empty?
+        # if @link_queue.empty? and @page_queue.empty?
+          if @page_queue.empty?
+           # if page_queue is empty wait for all grabber threads finishing the link_queue
             until @link_queue.num_waiting == @grabber_threads.length
               Thread.pass
             end
+            # when the link_queue is finished check the page_queue. Crawling is finished if page_queue is empty too.
             if @page_queue.empty?
               @grabber_threads.each { |t| t.kill }
               puts "Finished Crawling"
-              @engine_status = CRAWL_NONE
+              @status_lock.synchronize{ @engine_status = CRAWL_NONE }
               notify(:log, "Crawling finished")
               notify( :update_status, current_status )
               break
@@ -314,17 +330,20 @@ end
 
    fp = "#{form.action}"
         fp << form.method
+        if form.request_data =~ /=/
         data = form.request_data.split("&").sort.join("&")
         if o[:clear_values]
           fp << data.gsub(/=[^&]*/,'=')
         else
         fp << data
         end
+        end
         fkey = Digest::MD5.hexdigest fp
         fkey
   end
 
   def send_form(form, depth)  
+    return false if @engine_status == CRAWL_NONE
   cfk = form_key(form, :clear_values => true)
   @form_counts[cfk] ||= 0
   @form_counts[cfk] += 1
@@ -335,7 +354,12 @@ end
   @form_keys[fk] = nil
   begin
     if @form_counts[cfk] < @opts[:max_repeat]
+      if form.buttons.length > 0
+        p = form.click_button
+      else
       p = form.submit()
+      end
+      puts p.class
       @page_queue.enq PageBag.new(p, depth+1)
     else
       puts "! MAX REPEAT !\nSkipped Form #{form.action}"
@@ -348,6 +372,7 @@ end
 
   def send_form?(form)
    # puts "SEND FORM?"
+     return false unless engine_running?
         return false unless @opts[:submit_forms] == true
        # puts "> submit_forms"
         return false unless allowed? form.action
@@ -416,7 +441,7 @@ end
         # need to handle different link objects, Mechanize::Page::Link and URIs
         uri = nil
           uri = link.uri if link.respond_to? :uri
-          uri = URI.parse(link)  if link.is_a? String
+          uri = URI.parse(link) if link.is_a? String
           uri = link if link.is_a? URI::HTTP
         
         return false if uri.nil?
@@ -450,6 +475,13 @@ end
    Digest::MD5.hexdigest key
  end
  
+ def engine_running?
+   @status_lock.synchronize do 
+     return false if @engine_status == CRAWL_NONE
+     return true
+   end
+ end
+ 
       def link_is_followed?(link)
         
         return true if @link_keys.has_key? link_key(link)

data/plugins/crawler/lib/grabber.rb

@@ -46,7 +46,7 @@ module Watobo
           return nil if page.nil?
           return PageBag.new( page, linkbag.depth+1 )
         rescue => bang
-          puts bang
+          puts bang #if $DEBUG
           puts bang.backtrace if $DEBUG
         end
         return nil
@@ -54,15 +54,18 @@ module Watobo
 
       def run
         Thread.new(@link_queue, @page_queue){ |lq, pq|
-          begin
-            loop do
-              link, referer, depth = lq.deq
+          loop do
+            begin
+              #link, referer, depth = lq.deq
+              link = lq.deq
+              next if link.depth > @opts[:max_depth]
               page = get_page(link)
               pq << page unless page.nil?
+
+            rescue => bang
+              puts bang
+              puts bang.backtrace
             end
-          rescue => bang
-            puts bang
-            puts bang.backtrace
           end
         }
       end
@@ -73,6 +76,7 @@ module Watobo
         @opts = opts
         begin
           @agent = Crawler::Agent.new(@opts)
+    
         rescue => bang
           puts bang
           puts bang.backtrace

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: watobo
 version: !ruby/object:Gem::Version
-  version: 0.9.9.pre1
+  version: 0.9.9.pre2
   prerelease: 6
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-29 00:00:00.000000000 Z
+date: 2012-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -331,7 +331,6 @@ files:
 - icons/Yellow Ball_24x24.ico
 - README
 - CHANGELOG
-- .yardopts
 - bin/watobo_gui.rb
 - bin/watobo
 homepage: http://watobo.sourceforge.net

data/.yardopts

@@ -1,24 +0,0 @@
-# .
-# .yardopts
-# 
-# Copyright 2012 by siberas, http://www.siberas.de
-# 
-# This file is part of WATOBO (Web Application Tool Box)
-#        http://watobo.sourceforge.com
-# 
-# WATOBO is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation version 2 of the License.
-# 
-# WATOBO is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License
-# along with WATOBO; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-# .
---no-private
-*.xxx
-- CHANGELOG