watobo 0.9.20 → 0.9.21

data/CHANGELOG.md

@@ -1,3 +1,55 @@
+Version 0.9.21
+===
+
+News
+---
+**General**
+
+* greatly increased speed of session import by using Marshal serialization instead of YAML for storing chats/findings.
+
+  **NOTE:** older formats will still work but they will be converted to .mrs files.
+  
+* added scanner setting 'ignore server errors' which (if enabled) will handle 5xx response codes as 'file does not exist'. It overrules custom error filters.
+* introduced the watobo info page. accessible via http://watobo - where you can download the CA cert as well as some installation information
+* the 'Define Target Scope' dialog now has a text filter  
+ 
+**Plugins**
+
+* added Adobe Experience Manager Enumeration, crawles the site by using information of AEM/CQ5 json-Extensions
+
+**Fuzzer**
+
+* results are now saved in yaml format
+
+**Modules**
+
+* new CQ5 modules for selectors and extensions detection
+* new CQ5 module for userenumeration via json and xml extensions
+
+**CustomViewer**
+
+* updating view after handler is loaded, no more tab/chat switch necessary
+
+Fixes
+---
+**General**
+
+* changed encoding in conversation table
+* Response.header_value parsing
+
+**Plugins**
+
+* [sqlmap] enhanced path detection to also match 'sqlmap' without .py extension
+
+**Fuzzer**
+
+* multiple fixes in statistics view
+
+**ConversationTable**
+
+* fixed autoscroll checkbox, no more autoscrolling if unchecked
+* fixed crash on Goto-Function (shift-g shortcut)
+
 Version 0.9.20
 ===
 
@@ -36,12 +88,12 @@ News
  
 **SSL-Checker**
  
- * optimized ssl checks - but keep in mihd number of checked ciphers depends on installed ruby version :/
+ * optimized ssl checks - but keep in mind that the number of checked ciphers depends on your ruby version
 
 Fixes
 ---
 
-** General **
+**General**
  
  * post parameter values containing equal signs ('=') will no longer be truncated
  

data/README.md

@@ -1,6 +1,6 @@
 WATOBO - THE Web Application Toolbox
 ===
-WATOBO is a security tool for testing web applications. It is intended to enable security professionals to perform efficient (semi-automated) web application security audits.
+WATABO is a security tool for testing web applications. It is intended to enable security professionals to perform efficient (semi-automated) web application security audits.
 
 Most important features:
 

data/config/scanner.yml

@@ -1,6 +1,7 @@
 :logout_signatures: 
 - ^Location.*login
 :smart_scan: true
+:ignore_server_errors: true 
 :custom_error_patterns: []
 :excluded_chats: []
 :max_parallel_checks: 15

data/custom-views/prettify-json.rb

@@ -0,0 +1,19 @@
+#.
+# prettify-json.rb
+#.
+# Copyright 2014 by siberas, http://www.siberas.de
+# This file is part of WATOBO (Web Application Tool Box) http://watobo.sourceforge.com
+# WATOBO is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License.
+# WATOBO is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+# You should have received a copy of the GNU General Public License along with WATOBO; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+lambda{|response|
+  begin
+    jb = JSON.parse(response.body.to_s)
+    out = JSON.pretty_generate jb
+  rescue => bang
+    out = "Could prettify response :(\n\n"
+    out << bang.to_s
+  end
+  out  
+}

data/lib/watobo/adapters/file/marshal_store.rb

@@ -0,0 +1,297 @@
+#.
+# marshal_store.rb
+#.
+# Copyright 2014 by siberas, http://www.siberas.de
+# This file is part of WATOBO (Web Application Tool Box) http://watobo.sourceforge.com
+# WATOBO is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License.
+# WATOBO is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+# You should have received a copy of the GNU General Public License along with WATOBO; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+# @private 
+module Watobo#:nodoc: all
+  class FileSessionStore < SessionStore
+    def num_chats
+      get_file_list(@conversation_path, "*-chat*").length
+    end
+
+    def num_findings
+      get_file_list(@findings_path, "*-finding*").length
+    end
+
+    def add_finding(finding)
+      return false unless finding.respond_to? :request
+      return false unless finding.respond_to? :response
+
+      finding_file = File.join("#{@findings_path}", "#{finding.id}-finding.mrs")
+      unless File.exists?(finding_file)
+        save_finding(finding_file, finding)       
+        return true
+      end
+      false
+    end
+
+    def delete_finding(finding)
+      finding_file = File.join("#{@findings_path}", "#{finding.id}-finding")
+      File.delete finding_file if File.exist? finding_file
+      file = finding_file + ".yml"
+      File.delete file if File.exist? file
+      file = finding_file + ".mrs"
+      File.delete file if File.exist? file
+
+    end
+    
+    def save_finding(fname, finding)
+      File.open(fname, 'wb'){|f|
+        f.print Marshal::dump(finding.to_h)
+        }
+    end
+
+    def update_finding(finding)
+      finding_file = File.join("#{@findings_path}", "#{finding.id}-finding.mrs")
+      
+      if File.exists?(finding_file) then
+        save_finding(finding_file, finding)
+      end
+
+    end
+
+    # add_scan_log
+    # adds a chat to a specific log store, e.g. if you want to log scan results.
+    # needs a scan_name (STRING) as its destination which will be created
+    # if the scan name does not exist.
+    def add_scan_log(chat, scan_name = nil)
+      return false unless chat.respond_to? :request
+      return false unless chat.respond_to? :response
+      begin
+      
+        return false if scan_name.nil?
+        return false if scan_name.empty?
+        
+        scan_name_clean = scan_name.gsub(/[:\\\/\.]*/,"_")
+        # puts ">> scan_name"
+        path = File.join(@scanlog_path, scan_name_clean)
+
+        Dir.mkdir path unless File.exist? path
+
+        file = File.join( path, "log_" + Time.now.to_f.to_s + ".mrs")
+
+        unless File.exists?(file)
+          File.open(file, "wb") { |fh|
+            fh.print Marshal::dump(chat.to_h)
+          }
+        end
+      
+        return true
+      rescue => bang
+        puts bang
+        puts bang.backtrace if $DEBUG
+      end
+      return false
+    end
+
+    def add_chat(chat)
+      return false unless chat_valid? chat
+      chat_file = File.join("#{@conversation_path}", "#{chat.id}-chat.mrs")
+      
+      unless File.exists?(chat_file)
+        File.open(chat_file, "wb") { |fh|
+          fh.print Marshal::dump(chat.to_h)
+        }
+      chat.file = chat_file
+      return true
+      end
+      return false
+    end
+
+    def each_chat(&block)
+      list = get_file_list(@conversation_path, "*-chat*")
+      list.each do |fname|
+        #puts 
+        chat = nil
+        if fname =~ /\.mrs$/
+          chat = Watobo::Utils.loadChatMarshal(fname)
+        elsif fname =~ /\.yml$/
+          chat = Watobo::Utils.loadChatYAML(fname) unless list.include?(fname.gsub(/yml$/,'mrs'))    
+        end
+        next if chat.nil?
+        yield chat if block_given?
+      end
+    end
+
+    def each_finding(&block)
+      list = get_file_list(@findings_path, "*-finding*")
+      list.each do |fname|
+        f = nil
+        if fname =~ /\.mrs$/
+          f = Watobo::Utils.loadFindingMarshal(fname)
+        elsif fname =~ /\.yml$/
+          f = Watobo::Utils.loadFindingYAML(fname) unless list.include?(fname.gsub(/yml$/,'mrs'))
+        end
+        next if f.nil?
+        yield f if block_given?
+      end
+    end
+
+    def initialize(project_name, session_name)
+
+      wsp = Watobo.workspace_path
+      return false unless File.exist? wsp
+      puts "* using workspace path: #{wsp}" if $DEBUG
+
+      @log_file = nil
+      @log_lock = Mutex.new
+
+      @project_path = File.join(wsp, project_name)
+      unless File.exist? @project_path
+        puts "* create project path: #{@project_path}" if $DEBUG
+        Dir.mkdir(@project_path)
+      end
+
+      @project_config_path = File.join(@project_path, ".config")
+      Dir.mkdir @project_config_path unless File.exist? @project_config_path
+
+      @session_path = File.join(@project_path, session_name)
+
+      unless File.exist? @session_path
+        puts "* create session path: #{@session_path}" if $DEBUG
+        Dir.mkdir(@session_path)
+      end
+
+      @session_config_path = File.join(@session_path, ".config")
+      Dir.mkdir @session_config_path unless File.exist? @session_config_path
+
+      sext = Watobo::Conf::General.session_settings_file_ext
+
+      @session_file = File.join(@session_path, session_name + sext)
+      @project_file = File.join(@project_path, project_name + Watobo::Conf::General.project_settings_file_ext)
+
+      @conversation_path = File.expand_path(File.join(@session_path, Watobo::Conf::Datastore.conversations))
+
+      @findings_path = File.expand_path(File.join(@session_path, Watobo::Conf::Datastore.findings))
+      @log_path = File.expand_path(File.join(@session_path, Watobo::Conf::Datastore.event_logs_dir))
+      @scanlog_path = File.expand_path(File.join(@session_path, Watobo::Conf::Datastore.scan_logs_dir))
+
+      [ @conversation_path, @findings_path, @log_path, @scanlog_path ].each do |folder|
+        if not File.exists?(folder) then
+          puts "create path #{folder}"
+          begin
+            Dir.mkdir(folder)
+          rescue SystemCallError => bang
+            puts "!!!ERROR:"
+            puts bang
+          rescue => bang
+            puts "!!!ERROR:"
+            puts bang
+          end
+        end
+      end
+
+      @log_file = File.join(@log_path, session_name + ".log")
+
+    #     @chat_files = get_file_list(@conversation_path, "*-chat")
+    #     @finding_files = get_file_list(@findings_path, "*-finding")
+    end
+
+    def save_session_settings(group, session_settings)
+      # puts ">> save_session_settings <<"
+      file = Watobo::Utils.snakecase group.gsub(/\.yml/,'')
+      file << ".yml"
+
+      session_file = File.join(@session_config_path, file)
+      # puts "Dest.File: #{session_file}"
+      #  puts session_settings.to_yaml
+      # puts "---"
+      Watobo::Utils.save_settings(session_file, session_settings)
+    end
+
+    def load_session_settings(group)
+      # puts ">> load_session_settings : #{group}"
+      file = Watobo::Utils.snakecase group.gsub(/\.yml/,'')
+      file << ".yml"
+
+      session_file = File.join(@session_config_path, file)
+      # puts "File: #{session_file}"
+      #  puts "---"
+
+      s = Watobo::Utils.load_settings(session_file)
+      s
+    end
+
+    def save_project_settings(group, project_settings)
+      # puts ">> save_project_settings : #{group}"
+      file = Watobo::Utils.snakecase group.gsub(/\.yml/,'')
+      file << ".yml"
+
+      project_file = File.join(@project_config_path, file)
+      # puts "Dest.File: #{project_file}"
+      # puts project_settings.to_yaml
+      # puts "---"
+      Watobo::Utils.save_settings(project_file, project_settings)
+
+    end
+
+    def load_project_settings(group)
+      # puts ">> load_project_settings : #{group}"
+      file = Watobo::Utils.snakecase group.gsub(/\.yml/,'')
+      file << ".yml"
+
+      project_file = File.join(@project_config_path, file)
+      # puts "File: #{project_file}"
+      # puts "---"
+
+      s = Watobo::Utils.load_settings(project_file)
+      s
+
+    end
+
+    def logs
+      l = ''
+      @log_lock.synchronize do
+        l = File.open(@log_file).read
+      end
+      l
+    end
+
+    def logger( message, prefs = {} )
+      opts = { :sender => "unknown", :level => Watobo::Constants::LOG_INFO }
+      opts.update prefs
+      return false if @log_file.nil?
+      begin
+        t = Time.now
+        now = t.strftime("%m/%d/%Y @ %H:%M:%S")
+        log_message = [ now ]
+        log_message << "#{opts[:sender]}"
+        if message.is_a? Array
+          log_message << message.join("\n| ")
+          log_message << "\n-"
+        else
+        log_message << message
+        end
+        @log_lock.synchronize do
+          File.open(@log_file,"a") do |lfh|
+            lfh.puts log_message.join("|")
+          end
+        end
+      rescue => bang
+        puts bang
+      end
+
+    end
+
+    private
+
+    def chat_valid?(chat)
+      return false unless chat.respond_to? :request
+      return false unless chat.respond_to? :response
+      true
+    end
+
+    def get_file_list(path, pattern)
+      fl = Dir["#{path}/#{pattern}"].sort_by{ |x| File.basename(x).sub(/[^0-9]*/,'').to_i }
+      
+      fl
+    end
+
+  end
+
+end

data/lib/watobo/adapters.rb

@@ -11,4 +11,5 @@
   require "watobo/adapters/#{lib}"
 end
 
-require "watobo/adapters/file/file_store"
+#require "watobo/adapters/file/file_store"
+require "watobo/adapters/file/marshal_store"

data/lib/watobo/core/active_check.rb

@@ -250,6 +250,10 @@ module Watobo#:nodoc: all
         return false if status.empty?
         return true, t_request, t_response if status =~ /^403/
         return false, t_request, t_response if status =~ /^40\d/
+        if status =~ /^50\d/
+         # puts "* ignore server errors #{Watobo::Conf::Scanner.ignore_server_errors.class}"
+          return false, t_request, t_response if Watobo::Conf::Scanner.ignore_server_errors
+        end
 
         #puts @settings[:custom_error_patterns] 
 

data/lib/watobo/core/chat.rb

@@ -60,6 +60,14 @@ module Watobo#:nodoc: all
 
     def source()
       @settings[:source]
+    end
+    
+    def to_h
+      h = {}
+      h.update @settings
+      h[:request] = @request.to_a
+      h[:response] = @response.to_a
+      h
     end
 
 

data/lib/watobo/core/chats.rb

@@ -53,7 +53,8 @@ module Watobo#:nodoc: all
       matches = []
       @chats.each do |c|
         if c.request.site == site then
-          matches.push c if o[:dir] == c.request.dir
+          matches.push c if o[:dir] == c.request.dir
+          yield c if block_given?
         end
         return matches if o[:max_count] > 0 and matches.length >= o[:max_count]
       end