watobo 0.9.8.724 → 0.9.9.pre1

data/plugins/crawler/crawler.rb

@@ -0,0 +1,79 @@
+# .
+# crawler.rb
+# 
+# Copyright 2012 by siberas, http://www.siberas.de
+# 
+# This file is part of WATOBO (Web Application Tool Box)
+#        http://watobo.sourceforge.com
+# 
+# WATOBO is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation version 2 of the License.
+# 
+# WATOBO is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with WATOBO; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+# .
+begin
+  require 'mechanize'
+rescue LoadError
+  puts "To use the crawler plugin you must install the 'mechanize' gem."
+  puts "Simply enter the command 'gem install mechanize'"
+end
+
+
+
+
+path = File.expand_path(File.dirname(__FILE__))
+
+%w( constants bags grabber engine uri_mp).each do |l|
+  require File.join(path, "lib", l)
+end
+ 
+if $0 == __FILE__
+  module Watobo
+    module Conf
+      class Interceptor
+        def self.port
+ #         8081
+        nil
+        end
+      end
+    end
+  end
+  
+  require 'yaml'
+  if ARGV.length > 0
+    url = ARGV[0]
+  end
+ 
+  hook = lambda{ |agent, request|
+    begin
+      puts request.class
+      puts request.method
+      puts request.methods.sort
+     
+      exit
+      clean_jar = Mechanize::CookieJar.new
+      agent.cookie_jar.each{|cookie|
+        puts "Cookie: #{cookie.name}"
+        clean_jar.add! cookie unless cookie.name =~ /^box/i
+      }
+      exit unless agent.cookie_jar.empty?(request.url)
+      agent.cookie_jar = clean_jar
+    rescue => bang
+      puts bang
+      puts bang.backtrace
+    end
+
+  }
+  hook = nil
+ 
+  crawler = Watobo::Crawler::Engine.new
+  crawler.run(url, :pre_connect_hook => hook, :num_grabbers => 1 )
+end

data/plugins/crawler/gui/auth_frame.rb

@@ -0,0 +1,270 @@
+# .
+# auth_frame.rb
+# 
+# Copyright 2012 by siberas, http://www.siberas.de
+# 
+# This file is part of WATOBO (Web Application Tool Box)
+#        http://watobo.sourceforge.com
+# 
+# WATOBO is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation version 2 of the License.
+# 
+# WATOBO is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with WATOBO; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+# .
+module Watobo
+  module Plugin
+    module Crawler
+      class Gui
+        class FormFieldsTable < FXTable
+          def subscribe(event, &callback)
+            (@event_dispatcher_listeners[event] ||= []) << callback
+          end
+
+          def clearEvents(event)
+            @event_dispatcher_listener[event].clear
+          end
+
+          def update_fields
+            self.getNumRows.times do |i|
+              self.getItemData(i,0).value = self.getItemText(i,1)
+            end
+          end
+
+          def clear_fields
+            initTable
+          end
+
+          def initialize(owner, opts={})
+            super(owner, opts)
+
+            @event_dispatcher_listeners = Hash.new
+            initTable()
+
+            self.connect(SEL_COMMAND, method(:onTableClick))
+
+            self.columnHeader.connect(SEL_COMMAND) do |sender, sel, index|
+            # self.fitColumnsToContents(index)
+            end
+          end
+
+          def add_field(field)
+            lastRowIndex = self.getNumRows
+            self.appendRows(1)
+            self.setItemData(lastRowIndex, 0, field)
+            self.setItemText(lastRowIndex, 0, field.name)
+            self.getItem(lastRowIndex, 0).justify = FXTableItem::LEFT
+            self.setItemText(lastRowIndex, 1, field.value)
+            self.getItem(lastRowIndex, 1).justify = FXTableItem::LEFT
+
+          end
+          private
+
+          def onTableClick(sender, sel, item)
+            begin
+              row = item.row
+              self.selectRow(row, false)
+              self.startInput(row,1)
+            rescue => bang
+              puts bang
+            end
+          end
+
+          def notify(event, *args)
+            if @event_dispatcher_listeners[event]
+              @event_dispatcher_listeners[event].each do |m|
+                m.call(*args) if m.respond_to? :call
+              end
+            end
+          end
+
+          def initTable
+            self.clearItems()
+            self.setTableSize(0, 2)
+
+            self.setColumnText( 0, "Field" )
+            self.setColumnText( 1, "Value" )
+
+            self.rowHeader.width = 0
+            self.setColumnWidth(0, 100)
+            self.setColumnWidth(1, 100)
+          end
+        end
+
+        class AuthFrame < FXVerticalFrame
+          attr_accessor :crawler
+          def subscribe(event, &callback)
+            (@event_dispatcher_listeners[event] ||= []) << callback
+          end
+
+          def clearEvents(event)
+            @event_dispatcher_listener[event].clear
+          end
+
+          def to_h
+            a = case @auth_type_dt.value
+            when 0
+              {
+                :auth_type => :none
+              }
+            when 1
+              {
+                :auth_type => :basic,
+                :username => @basic_auth_user_txt.text,
+                :password => @basic_auth_passwd_txt.text
+                #  :uri => URI.parse
+              }
+            when 2
+              @form_fields_table.update_fields
+
+              {
+                :auth_type => :form,
+                :form => @auth_form,
+                :cookie_jar => @agent.respond_to?(:cookie_jar) ? @agent.cookie_jar : nil
+              }
+            end
+            a
+
+          end
+
+          def set(settings)
+            @form_auth_url_txt.text = settings[:form_auth_url].to_s if settings.has_key? :form_auth_url
+            update_form
+          end
+
+          def update_form
+            [   @form_auth_url_txt ].each do |e|
+              e.handle(self, FXSEL(SEL_UPDATE, 0), nil)
+            end
+          end
+
+          def initialize(owner, opts={})
+            super(owner, :opts => LAYOUT_FILL_X|LAYOUT_FILL_Y|FRAME_THICK|FRAME_RAISED, :padding => 0)
+            frame = FXHorizontalFrame.new(self, :opts => LAYOUT_FILL_X|LAYOUT_FILL_Y)
+            #left_frm = FXVerticalFrame.new(self, :opts => LAYOUT_FILL_Y|FRAME_NONE)
+            @event_dispatcher_listeners = Hash.new
+            @crawler = opts[:crawler] if opts.has_key? :crawler
+            @start_url = ""
+
+            auth_gb= FXGroupBox.new(frame, "Authentication", LAYOUT_SIDE_RIGHT|FRAME_GROOVE|LAYOUT_FILL_X|LAYOUT_FILL_Y, 0, 0, 0, 0)
+            auth_frm = FXVerticalFrame.new(auth_gb, :opts => LAYOUT_FILL_X|LAYOUT_FILL_Y|FRAME_NONE)
+            @auth_form = nil
+            @auth_type_dt = FXDataTarget.new(0)
+
+            @no_auth_rb = FXRadioButton.new(auth_frm, "None", @auth_type_dt, FXDataTarget::ID_OPTION)
+
+            @basic_auth_rb = FXRadioButton.new(auth_frm, "Basic", @auth_type_dt, FXDataTarget::ID_OPTION + 1)
+
+            @form_auth_rb = FXRadioButton.new(auth_frm, "Form", @auth_type_dt, FXDataTarget::ID_OPTION + 2)
+
+            # group_box = FXGroupBox.new(self, "Collection",LAYOUT_SIDE_TOP|FRAME_GROOVE|LAYOUT_FILL_X|LAYOUT_FILL_Y, 0, 0, 0, 0)
+            # frame = FXVerticalFrame.new(group_box, :opts => LAYOUT_FILL_X|LAYOUT_FILL_Y|FRAME_SUNKEN|FRAME_GROOVE)
+            # @collectionList = FXList.new(frame, :opts => LIST_EXTENDEDSELECT|LAYOUT_FILL_X|LAYOUT_FILL_Y)
+
+            @switcher = FXSwitcher.new(auth_frm,LAYOUT_FILL_X|LAYOUT_FILL_Y)
+            frame = FXHorizontalFrame.new(@switcher, :opts => LAYOUT_FILL_X|LAYOUT_FILL_Y|FRAME_NONE)
+            FXLabel.new(frame, "No Authentication Selected", nil, LAYOUT_TOP|JUSTIFY_RIGHT)
+            frame = FXHorizontalFrame.new(@switcher, :opts => LAYOUT_FILL_X|LAYOUT_FILL_Y|FRAME_NONE)
+            FXLabel.new(frame, "Username:", nil, LAYOUT_TOP|JUSTIFY_RIGHT)
+            @basic_auth_user_txt = FXTextField.new(frame, 10, nil, 0, :opts => TEXTFIELD_NORMAL|LAYOUT_SIDE_RIGHT)
+            FXLabel.new(frame, "Password:", nil, LAYOUT_TOP|JUSTIFY_RIGHT)
+            @basic_auth_passwd_txt = FXTextField.new(frame, 10, nil, 0, :opts => TEXTFIELD_NORMAL|LAYOUT_SIDE_RIGHT)
+            FXLabel.new(frame, "Retype:", nil, LAYOUT_TOP|JUSTIFY_RIGHT)
+            @basic_auth_retype_txt = FXTextField.new(frame, 10, nil, 0, :opts => TEXTFIELD_NORMAL|LAYOUT_SIDE_RIGHT)
+
+            form_auth_frame = FXVerticalFrame.new(@switcher, :opts => LAYOUT_FILL_X|LAYOUT_FILL_Y|FRAME_NONE)
+            frame = FXHorizontalFrame.new(form_auth_frame, :opts => LAYOUT_FILL_X|FRAME_NONE)
+            FXLabel.new(frame, "URL of LoginForm, leave empty to use Start URL:", nil, LAYOUT_TOP|JUSTIFY_RIGHT)
+            @form_auth_url_txt = FXTextField.new(frame, 10, nil, 0, :opts => TEXTFIELD_NORMAL|LAYOUT_FILL_X)
+            @fetch_button = FXButton.new(frame, "load page", :opts => BUTTON_DEFAULT|BUTTON_NORMAL )
+
+            form_frame = FXHorizontalFrame.new(form_auth_frame, :opts => LAYOUT_FILL_X|LAYOUT_FILL_Y|FRAME_NONE)
+            frame = FXHorizontalFrame.new(form_frame, :opts => LAYOUT_FILL_X|LAYOUT_FILL_Y|FRAME_SUNKEN|FRAME_THICK, :padding => 0)
+            @page_tree = Watobo::Gui::PageTree.new(frame)
+            frame = FXHorizontalFrame.new(form_frame, :opts => LAYOUT_FILL_X|LAYOUT_FILL_Y|FRAME_THICK)
+            @form_fields_table = FormFieldsTable.new(frame, :opts => LAYOUT_FILL_X|LAYOUT_FILL_Y)
+
+            @auth_type_dt.connect(SEL_COMMAND) {
+              @basic_auth_rb.handle(self, FXSEL(SEL_UPDATE, 0), nil)
+              @form_auth_rb.handle(self, FXSEL(SEL_UPDATE, 0), nil)
+              @no_auth_rb.handle(self, FXSEL(SEL_UPDATE, 0), nil)
+              @switcher.current = @auth_type_dt.value
+            }
+
+            @fetch_button.connect(SEL_COMMAND){
+              begin
+                @form_fields_table.clear_fields
+                @page_tree.clearItems
+
+                page = nil
+                url = nil
+                unless Watobo::Plugin::Crawler.start_url.nil?
+                  uri = Watobo::Plugin::Crawler.start_url
+                else
+                  url = @form_auth_url_txt.text unless @form_auth_url_txt.text.empty?
+                  uri = URI.parse(url) 
+                end
+
+                notify(:log, "GET PAGE << #{url}")
+                @agent, page = @crawler.get_page(uri)
+                notify(:log, "PAGE LOADED")
+                @page_tree.page = page
+              rescue => bang
+                notify(:log, "could not get page #{url}")
+                puts "Could not get page for #{url}"
+                puts url.class
+                puts bang
+                puts bang.backtrace
+              end
+              true
+            }
+            @switcher.current = @auth_type_dt.value
+
+            @page_tree.subscribe(:form_selected){|form|
+              @auth_form = form
+              @form_fields_table.clear_fields
+              form.fields.each do |field|
+                @form_fields_table.add_field field
+              end
+            }
+
+          end
+
+          private
+
+          def notify(event, *args)
+            if @event_dispatcher_listeners[event]
+              @event_dispatcher_listeners[event].each do |m|
+                m.call(*args) if m.respond_to? :call
+              end
+            end
+          end
+
+          def enable_basic_auth
+
+          end
+
+          def disable_basic_auth
+
+          end
+
+          def enable_form_auth
+
+          end
+
+          def disable_form_auth
+
+          end
+
+        end
+      end
+    end
+  end
+end

data/plugins/crawler/gui/crawler_gui.rb

@@ -0,0 +1,252 @@
+# .
+# crawler_gui.rb
+# 
+# Copyright 2012 by siberas, http://www.siberas.de
+# 
+# This file is part of WATOBO (Web Application Tool Box)
+#        http://watobo.sourceforge.com
+# 
+# WATOBO is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation version 2 of the License.
+# 
+# WATOBO is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with WATOBO; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+# .
+module Watobo
+  module Plugin
+    module Crawler
+      def self.start_url
+        @start_url
+      end
+
+      def self.start_url=(url)
+        @start_url ||= ""
+        host = nil
+        begin
+          host = URI.parse(url).host
+        rescue => bang
+          host = nil
+        end
+        @start_url = url unless host.nil?
+        @start_url
+      end
+
+      class Gui < Watobo::Plugin2
+        icon_file "crawler.ico"
+
+        include Watobo::Constants
+        include Watobo::Plugin::Crawler::Constants
+        def updateView
+
+        end
+
+        def start_url
+          url = url_valid? ? URI.parse(@url_txt.text) : nil
+          return url
+        end
+
+        def settings
+          @settings_tabbook
+        end
+
+        def set_tab_index(index)
+          @settings_tabbook.setCurrent index
+        end
+
+        def initialize(owner, project=nil)
+          super(owner, "Crawler", project, :opts => DECOR_ALL, :width=>800, :height=>600)
+          @plugin_name = "Crawler"
+          @project = project
+          @status_lock = Mutex.new
+          @crawl_status = {
+            :engine_status => CRAWL_NONE,
+            :page_size => 0,
+            :link_size => 0,
+            :skipped_domains => 0
+          }
+
+          main = FXVerticalFrame.new(self, :opts => LAYOUT_FILL_X|LAYOUT_FILL_Y)
+          FXLabel.new(main, "Start URL, e.g. http://my.target.to/scan/:")
+          frame = FXHorizontalFrame.new(main, :opts => LAYOUT_FILL_X)
+          #  FXLabel.new(frame, "http://")
+          @url_txt = FXTextField.new(frame, 60, nil, 0, :opts => TEXTFIELD_NORMAL|LAYOUT_SIDE_RIGHT|LAYOUT_FILL_X)
+
+          @start_button = FXButton.new(frame, "start", :opts => BUTTON_DEFAULT|BUTTON_NORMAL )
+          @start_button.disable
+
+          @url_txt.connect(SEL_COMMAND){ |sender, sel, item|
+            @start_button.setFocus() if url_valid?
+          }
+
+          @url_txt.connect(SEL_CHANGED){
+            if url_valid?
+              Watobo::Plugin::Crawler.start_url = start_url
+            @start_button.enable
+            else
+              @start_button.disable
+              Watobo::Plugin::Crawler.start_url = nil
+            end
+          }
+
+          @start_button.connect(SEL_COMMAND){ |sender, sel, item|
+            case sender.text
+            when /start/i
+              start
+            when /cancel/i
+              cancel
+            end
+          }
+
+          @crawler = Watobo::Crawler::Engine.new
+
+          @settings_tabbook = SettingsTabBook.new(main)
+          @settings_tabbook.general.set @crawler.settings
+          @settings_tabbook.auth.crawler = @crawler
+          @settings_tabbook.scope.set @crawler.settings
+
+          @log_viewer = @settings_tabbook.log_viewer
+
+          @status_frame = StatusFrame.new(main)
+
+          @crawler.subscribe( :update_status ){ |status|
+            @status_lock.synchronize do
+              @crawl_status.update status
+            end
+          }
+
+          stbk = @settings_tabbook
+          [ @crawler, stbk.auth].each do |i|
+            i.subscribe( :log ){ |msg|
+              @log_viewer.log(LOG_INFO, msg)
+            }
+          end
+
+        end
+
+        private
+
+        def remove_update_timer
+          app = FXApp.instance
+          if app.hasTimeout? @update_timer
+          app.removeTimeout @update_timer
+          end
+        end
+
+        def add_update_timer(ms=50)
+          @update_timer = FXApp.instance.addTimeout( ms, :repeat => true) {
+           update_status
+          }
+
+        end
+
+        def update_status
+          @status_lock.synchronize do
+
+            @status_frame.update_status @crawl_status
+
+            if @crawl_status.has_key? :engine_status
+              case @crawl_status[:engine_status]
+              when CRAWL_NONE
+                @start_button.text = "start"
+              when CRAWL_RUNNING
+                @start_button.text = "cancel"
+
+              when CRAWL_PAUSED
+                @start_button.text = "start"
+              end
+            end
+
+          end
+        end
+
+        def cancel
+          remove_update_timer()
+          @crawler.cancel
+          @start_button.text = "start"
+          update_status
+        end
+
+        def auth_settings()
+          puts "= Authentication Settings ="
+          auth = @settings_tabbook.auth.to_h
+          # puts auth.to_yaml
+          case auth[:auth_type]
+          when :form
+            if auth.has_key? :form
+              #puts ""
+              #@crawler.send_form(form)
+              if auth[:form].respond_to? :submit
+                puts "Submitting Form"
+                p = auth[:form].submit()
+                puts p.class
+              end
+            end
+          end
+          puts "---"
+          auth
+        end
+
+        def scope_settings()
+          # puts "= Scope Settings ="
+          scope = @settings_tabbook.scope
+          ss = scope.to_h
+          ss[:root_path] = start_url.path_mp if scope.path_restricted?
+          ss
+        end
+
+        def general_settings()
+          #  puts "= General Settings ="
+
+          gs = @settings_tabbook.general.to_h
+
+        end
+
+        def hook_settings
+          hs = @settings_tabbook.hooks.to_h
+        end
+
+        def url_valid?
+          begin
+            return false unless @url_txt.text =~ /^https?:\/\//
+            url = URI.parse(@url_txt.text)
+            # puts url.host.class
+            return true unless url.host.nil?
+            return false
+          rescue => bang
+            puts bang if $DEBUG
+            return false
+          end
+        end
+
+        def start
+          return false unless url_valid?
+
+          prefs ={}
+          prefs.update auth_settings
+          prefs.update scope_settings
+          prefs.update general_settings
+          prefs.update hook_settings
+
+          # puts prefs.to_yaml
+
+          Thread.new{
+            url = @url_txt.text
+            @crawler.run(url, prefs)
+          }
+
+          @start_button.text = 'Cancel'
+          add_update_timer()
+        end
+
+      end
+
+    end
+  end
+end