watobo 0.9.21 → 0.9.23

data/plugins/aem/lib/agent.rb

@@ -1,140 +1,132 @@
-#.
-# agent.rb
-#.
-# Copyright 2014 by siberas, http://www.siberas.de
-# This file is part of WATOBO (Web Application Tool Box) http://watobo.sourceforge.com
-# WATOBO is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License.
-# WATOBO is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
-# You should have received a copy of the GNU General Public License along with WATOBO; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+# @private 
+module Watobo#:nodoc: all
+  module Plugin
+    class CQ5
+      class Agent < Watobo::Session
+         def initialize(base_request, in_queue, out_queue )
+            @work_queue = in_queue
+            @disp_queue = out_queue
+            @request = base_request
+            
+            
+            super(@request.object_id,  Watobo::Conf::Scanner.to_h )
 
-# @private 
-module Watobo#:nodoc: all
-  module Plugin
-    class CQ5
-      class Agent < Watobo::Session
-         def initialize(base_request, in_queue, out_queue )
-            @work_queue = in_queue
-            @disp_queue = out_queue
-            @request = base_request
-            
-            
-            super(@request.object_id,  Watobo::Conf::Scanner.to_h )
-
-         end
-         
-         def stop
-           @agent_thread.kill
-         end
-         
-         def run
-           return nil if @work_queue.nil? or @disp_queue.nil?
-           
-           @agent_thread = Thread.new(){
-             puts "#{self} running ..." 
-             loop do
-             begin
-              
-               item = @work_queue.deq
-               # not interested in jcr:content ... skip ... 
-               next if item[:url] =~ /jcr%3acontent$/
-              
-               get_pages item
-               file_info item
-               
-             rescue => bang
-               puts bang
-               puts bang.backtrace
-               exit
-             end
-             end
-           }
-           @agent_thread
-         end
-         
-         def get_pages(item)           
-           test = @request.copy
-          # puts item
-           url = item[:url].gsub(/\/$/,'') + '/.pages.json'
-           test.replaceURL( url )
-           
-           request, response = sendRequest test
-           
-           return false unless response.respond_to? :status
-           item[:pages_status] = response.status
-          # @disp_queue << item
-             
-                  
-           if response.content_type =~ /json/i
-             begin
-             ntpages = JSON.parse response.body.to_s
-             
-             if ntpages['pages']
-                            
-              ntpages['pages'].each do |p|
-                #unless @use_relative_path
-                ep = p['escapedPath']
-                next if ep.nil?
-                next if ep.empty?
-                 
-                purl = ''
-               
-                purl = @request.url.to_s.gsub(/\/$/, '') + p['escapedPath']
-                puts "+ #{purl}"
-                
-                next if purl.empty?
-                
-                item = {
-                  :url => purl,
-                  :page_info => p,
-                  :file_info => nil,
-                  :status => nil
-                }
-                                 
-                @disp_queue << item
-               end              
-             end      
-           
-             rescue => bang
-                puts bang
-                puts ntpages
-                puts "---"
-              end   
-           end
-           #puts response.body.to_s 
-           true
-         end
-         
-         def file_info(item)
-           #url = item[:url]
-           #@request.replaceURL "#{url}/.json"
-           test = @request.copy
-           test.set_file_extension "json"
-          # puts "\n>> #{@request.url}"
-           request, response = sendRequest test
-           
-           return false unless response.respond_to? :status
-           item[:info_status] = response.status
-           
-           if response.content_type =~ /json/i
-             info = JSON.parse response.body.to_s
-             item[:file_info] = info
-           end
-           #@disp_queue << item
-           true
-         end
-
-         def sendRequest(request, prefs={})
-           begin
-              test_req, test_resp = self.doRequest(request, prefs)
-              return test_req, test_resp
-           rescue => bang
-              puts bang
-              puts bang.backtrace if $DEBUG
-           end
-           return nil, nil            
-         end
-      end
-      
-    end
-  end
+         end
+         
+         def stop
+           @agent_thread.kill
+         end
+         
+         def run
+           return nil if @work_queue.nil? or @disp_queue.nil?
+           
+           @agent_thread = Thread.new(){
+             puts "#{self} running ..." 
+             loop do
+             begin
+              
+               item = @work_queue.deq
+               # not interested in jcr:content ... skip ... 
+               next if item[:url] =~ /jcr%3acontent$/
+               puts item[:url]
+              
+               get_pages item
+               file_info item
+               
+             rescue => bang
+               puts bang
+               puts bang.backtrace
+               exit
+             end
+             end
+           }
+           @agent_thread
+         end
+         
+         def get_pages(item)           
+           test = @request.copy
+          # puts item
+           url = item[:url].gsub(/\/$/,'') + '/.pages.json'
+           test.replaceURL( url )
+           
+           request, response = sendRequest test
+           
+           return false unless response.respond_to? :status
+           item[:pages_status] = response.status
+          # @disp_queue << item
+             
+                  
+           if response.content_type =~ /json/i
+             begin
+             ntpages = JSON.parse response.body.to_s
+             
+             if ntpages['pages']
+                            
+              ntpages['pages'].each do |p|
+                #unless @use_relative_path
+                ep = p['escapedPath']
+                next if ep.nil?
+                next if ep.empty?
+                 
+                purl = ''
+               
+                purl = @request.url.to_s.gsub(/\/$/, '') + p['escapedPath']
+                puts "+ #{purl}"
+                
+                next if purl.empty?
+                
+                item = {
+                  :url => purl,
+                  :page_info => p,
+                  :file_info => nil,
+                  :status => nil
+                }
+                                 
+                @disp_queue << item
+               end              
+             end      
+           
+             rescue => bang
+                puts bang
+                puts ntpages
+                puts "---"
+              end   
+           end
+           #puts response.body.to_s 
+           true
+         end
+         
+         def file_info(item)
+           #url = item[:url]
+           #@request.replaceURL "#{url}/.json"
+           test = @request.copy
+           test.set_file_extension "json"
+          # puts "\n>> #{@request.url}"
+           request, response = sendRequest test
+           
+           return false unless response.respond_to? :status
+           item[:info_status] = response.status
+           
+           if response.content_type =~ /json/i
+             info = JSON.parse response.body.to_s
+             item[:file_info] = info
+           end
+           #@disp_queue << item
+           true
+         end
+
+         def sendRequest(request, prefs={})
+           begin
+              test_req, test_resp = self.doRequest(request, prefs)
+              return test_req, test_resp
+           rescue => bang
+              puts bang
+              puts bang.backtrace if $DEBUG
+           end
+           return nil, nil            
+         end
+      end
+      
+    end
+  end
 end

data/plugins/aem/lib/dispatcher.rb

@@ -1,53 +1,47 @@
-#.
-# dispatcher.rb
-#.
-# Copyright 2014 by siberas, http://www.siberas.de
-# This file is part of WATOBO (Web Application Tool Box) http://watobo.sourceforge.com
-# WATOBO is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License.
-# WATOBO is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
-# You should have received a copy of the GNU General Public License along with WATOBO; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+# @private
+module Watobo #:nodoc: all
+  module Plugin
+    class CQ5
+      class Dispatcher
 
-# @private
-module Watobo#:nodoc: all
-  module Plugin
-    class CQ5
-      class Dispatcher
-        
-        def stop
-          @t_disp.kill unless @t_disp.nil?
-        end
-        
-        def run
-          @known_urls = []
-          puts Watobo::Plugin::CQ5.ignore_patterns
-          @t_disp = Thread.new{
-            loop do
-               new_item = @dqueue.deq
-               unless @known_urls.include?( new_item[:url] )
-                 @known_urls << new_item[:url]
-                 if Watobo::Plugin::CQ5.ignore_patterns.empty?
-                  # puts "* no ignore patterns defined"
-                   @wqueue << new_item
-                 elsif Watobo::Plugin::CQ5.ignore_patterns.select{|ip| new_item[:url] =~ /#{ip}/i }.empty?
-                   @wqueue << new_item 
-                 end
-                 
-                 @rqueue << new_item
-               else
-                 puts "[DUPLICATED] >> #{new_item[:url]}"
-               end
-              
-             end
-          }
-        end
-        
-        def initialize(disp_queue, work_queue, result_queue)
-          @dqueue = disp_queue
-          @wqueue = work_queue
-          @rqueue = result_queue
-          @t_disp = nil
-        end
-      end
-    end
-  end
+        def stop
+          @t_disp.kill unless @t_disp.nil?
+        end
+
+        def run
+          @known_urls = []
+          puts Watobo::Plugin::CQ5.ignore_patterns
+          @t_disp = Thread.new {
+            loop do
+              if @dqueue.size > 0
+                new_item = @dqueue.deq
+                unless @known_urls.include?(new_item[:url])
+                  @known_urls << new_item[:url]
+                  if Watobo::Plugin::CQ5.ignore_patterns.empty?
+                    # puts "* no ignore patterns defined"
+                    @wqueue << new_item
+                  elsif Watobo::Plugin::CQ5.ignore_patterns.select { |ip| new_item[:url] =~ /#{ip}/i }.empty?
+                    @wqueue << new_item
+                  end
+
+                  @rqueue << new_item
+                else
+                  puts "[DUPLICATED] >> #{new_item[:url]}"
+                end
+              else
+                sleep 0.2
+              end
+            end
+          }
+        end
+
+        def initialize(disp_queue, work_queue, result_queue)
+          @dqueue = disp_queue
+          @wqueue = work_queue
+          @rqueue = result_queue
+          @t_disp = nil
+        end
+      end
+    end
+  end
 end

data/plugins/aem/lib/engine.rb

@@ -1,187 +1,178 @@
-#.
-# engine.rb
-#.
-# Copyright 2014 by siberas, http://www.siberas.de
-# This file is part of WATOBO (Web Application Tool Box) http://watobo.sourceforge.com
-# WATOBO is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License.
-# WATOBO is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
-# You should have received a copy of the GNU General Public License along with WATOBO; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-
-# @private 
-module Watobo#:nodoc: all
-  module Plugin
-    class CQ5
-      @max_agents = 10
-      @disp_queue = Queue.new
-      @work_queue = Queue.new
-      @gui_queue = Queue.new
-      
-      @agents = []
-      @use_relative_path = false
-      
-      
-      def self.reset
-        @disp_queue.clear
-        @work_queue.clear
-        @agents.map {|a| a.stop }
-        @agents = []
-      end
-      
-      def self.ignore_patterns=(ipats)
-        @ignore_patterns = ipats
-      end
-      
-      def self.ignore_patterns
-        @ignore_patterns
-      end
-      
-            
-      def self.use_relative_path=(urp)
-        @use_relative_path = urp
-      end
-      
-      def self.queue_size
-        @work_queue.size
-      end
-      
-      def self.status
-        
-      end
-      
-      def self.running?
-        @work_queue.size == 0 &&
-        @work_queue.num_waiting == @max_agents        
-      end
-      
-      def self.stop
-          @agents.map {|a| a.stop }
-      end
-      
-      def self.run(start_path, gui_queue=nil)
-        @agents = []
-        puts "\nCQ5 Engine running on #{start_path}"
-        @dispatcher = Dispatcher.new( @disp_queue, @work_queue, gui_queue )
-        @dispatcher.run
-        
-        vr = find_valid_request(start_path)
-        unless vr.nil?
-          puts "Baseline Request: " + vr.url.to_s
-         
-          @max_agents.times do 
-            puts " * Start Agent"
-            a = Agent.new( vr.copy, @work_queue, @disp_queue )
-            @agents << a
-            a.run
-          end
-          
-          
-        else
-          return false
-        end
-        
-        
-      end
-      
-      def self.get_user_info
-        # https://mysite/cqa/libs/cq/security/userinfo.json?cq_ck=1427468388796
-      end
-      
-      def self.find_valid_request(start_path)
-        # create a dummy agent to make test requests
-        agent = Agent.new nil, nil, nil
-        checked = []
-        #puts start_path.class
-        valid_request = nil
-        
-        Watobo::Chats.to_a.reverse.each do |chat|
-          next unless chat.request.method_get?
-          url = chat.request.url.to_s
-        #  url.gsub!(chat.request.site, chat.request.host)
-          path = chat.request.path
-          
-        #  next if checked.include? path
-          
-          checked << path 
-         # puts path.class
-         # puts url.class
-         # puts start_path
-         # puts url
-          
-          pattern = Regexp.quote(start_path)
-          pattern = start_path
-          #puts pattern
-          
-          #puts "---\n"
- 
-          if url =~ /#{pattern}/
-            test = chat.copyRequest
-            test.replaceFileExt('.pages.json')
-            
-             
-            puts "* [#{chat.id}] " + test.url.to_s
-            
-            request, response = agent.doRequest test
-            
-           # puts response
-            
-            unless response.content_type =~ /json/i
-              puts "! .pages.json is filtered !"
-              next
-            end
-            ntpages = JSON.parse response.body.to_s
-            if ntpages['pages']
-              valid_request = test
-                            
-              ntpages['pages'].each do |p|
-                # check if escapedPath is absolut or relativ
-                # if we find directory separator '/' we assume it's absolute
-                ep = p['escapedPath'].gsub(/^\//,'').strip
-                puts "EscapedPath: #{ep}"
-                
-                # find the home directory of the application
-                ep_dirs = ep.split('/')
-                
-                puts "EscapedPath-Dirs (#{ep_dirs.length}): " + ep_dirs.join("\n")
-                # request dir
-                r_dir = "#{test.dir}"
-                puts "Test-Request-Dir: " + r_dir
-                puts "Check for #{ep_dirs[0]}, #{ep_dirs[1]}"
-                # find offset of first escapePath directory
-                i = r_dir.index( ep_dirs.first )
-                puts "Index: #{i}"
-                base_dir = r_dir
-                puts base_dir
-                unless i.nil?
-                  if i > 0
-                    base_dir = r_dir[0..i-1]
-                  else
-                    base_dir = ''
-                  end
-                end 
-                
-                puts "Base-Dir: #{base_dir}"
-                test.setDir base_dir
-                
-                item = {
-                  #:url => base_url.gsub(/\/$/,'') + p['escapedPath'],
-                  :url => start_path,
-                  :page_info => p,
-                  :file_info => nil,
-                  :status => nil
-                }
-                
-              
-                @disp_queue << item                 
-              
-              end
-              #test.replaceFileExt('')
-              return valid_request
-            end            
-          end
-        end
-        nil
-      end
-      
-    end
-  end
+# @private 
+module Watobo#:nodoc: all
+  module Plugin
+    class CQ5
+      @max_agents = 10
+      @disp_queue = Queue.new
+      @work_queue = Queue.new
+      @gui_queue = Queue.new
+      
+      @agents = []
+      @use_relative_path = false
+      
+      
+      def self.reset
+        @disp_queue.clear
+        @work_queue.clear
+        @agents.map {|a| a.stop }
+        @agents = []
+      end
+      
+      def self.ignore_patterns=(ipats)
+        @ignore_patterns = ipats
+      end
+      
+      def self.ignore_patterns
+        @ignore_patterns
+      end
+      
+            
+      def self.use_relative_path=(urp)
+        @use_relative_path = urp
+      end
+      
+      def self.queue_size
+        @work_queue.size
+      end
+      
+      def self.status
+        
+      end
+      
+      def self.running?
+        @work_queue.size == 0 &&
+        @work_queue.num_waiting == @max_agents        
+      end
+      
+      def self.stop
+          @agents.map {|a| a.stop }
+      end
+      
+      def self.run(start_path, gui_queue=nil)
+        @agents = []
+        puts "\nCQ5 Engine running on #{start_path}"
+        @dispatcher = Dispatcher.new( @disp_queue, @work_queue, gui_queue )
+        @dispatcher.run
+        
+        vr = find_valid_request(start_path)
+        unless vr.nil?
+          puts "Baseline Request: " + vr.url.to_s
+         
+          @max_agents.times do 
+            puts " * Start Agent"
+            a = Agent.new( vr.copy, @work_queue, @disp_queue )
+            @agents << a
+            a.run
+          end
+          
+          
+        else
+          return false
+        end
+        
+        
+      end
+      
+      def self.get_user_info
+        # https://mysite/cqa/libs/cq/security/userinfo.json?cq_ck=1427468388796
+      end
+      
+      def self.find_valid_request(start_path)
+        # create a dummy agent to make test requests
+        agent = Agent.new nil, nil, nil
+        checked = []
+        #puts start_path.class
+        valid_request = nil
+        
+        Watobo::Chats.to_a.reverse.each do |chat|
+          next unless chat.request.method_get?
+          url = chat.request.url.to_s
+        #  url.gsub!(chat.request.site, chat.request.host)
+          path = chat.request.path
+          
+        #  next if checked.include? path
+          
+          checked << path 
+         # puts path.class
+         # puts url.class
+         # puts start_path
+         # puts url
+          
+          pattern = Regexp.quote(start_path)
+          pattern = start_path
+          #puts pattern
+          
+          #puts "---\n"
+ 
+          if URI.parse(url).to_s =~ /#{pattern}/
+            test = chat.copyRequest
+            test.replaceFileExt('.pages.json')
+            
+             
+            puts "* [#{chat.id}] " + test.url.to_s
+            
+            request, response = agent.doRequest test
+            
+            puts response
+            
+            unless response.content_type =~ /json/i
+              puts "! .pages.json is filtered !"
+              next
+            end
+            ntpages = JSON.parse response.body.to_s
+            if ntpages['pages']
+              valid_request = test
+                            
+              ntpages['pages'].each do |p|
+                # check if escapedPath is absolut or relativ
+                # if we find directory separator '/' we assume it's absolute
+                ep = p['escapedPath'].gsub(/^\//,'').strip
+                puts "EscapedPath: #{ep}"
+                
+                # find the home directory of the application
+                ep_dirs = ep.split('/')
+                
+                puts "EscapedPath-Dirs (#{ep_dirs.length}): " + ep_dirs.join("\n")
+                # request dir
+                r_dir = "#{test.dir}"
+                puts "Test-Request-Dir: " + r_dir
+                puts "Check for #{ep_dirs[0]}, #{ep_dirs[1]}"
+                # find offset of first escapePath directory
+                i = r_dir.index( ep_dirs.first )
+                puts "Index: #{i}"
+                base_dir = r_dir
+                puts base_dir
+                unless i.nil?
+                  if i > 0
+                    base_dir = r_dir[0..i-1]
+                  else
+                    base_dir = ''
+                  end
+                end 
+                
+                puts "Base-Dir: #{base_dir}"
+                test.setDir base_dir
+                
+                item = {
+                  #:url => base_url.gsub(/\/$/,'') + p['escapedPath'],
+                  :url => start_path,
+                  :page_info => p,
+                  :file_info => nil,
+                  :status => nil
+                }
+                
+              
+                @disp_queue << item                 
+              
+              end
+              #test.replaceFileExt('')
+              return valid_request
+            end            
+          end
+        end
+        nil
+      end
+      
+    end
+  end
 end