wmap 2.7.6 → 2.8.2

data/lib/wmap/url_crawler.rb

@@ -17,7 +17,8 @@ require "parallel"
 class Wmap::UrlCrawler
 	include Wmap::Utils
 
-	attr_accessor :http_timeout, :crawl_page_limit, :crawl_depth, :max_parallel, :verbose, :data_dir
+	attr_accessor :http_timeout, :crawl_page_limit, :crawl_depth, :max_parallel, \
+				:verbose, :data_dir, :user_agent
 	attr_reader :discovered_urls_by_crawler, :visited_urls_by_crawler, :crawl_start, :crawl_done
 	# Global variable used to store the combined result of all the forked child processes. Note that class variable
 	# would not be able to pass the result due the limitation of IO Pipe communication mechanism used by 'parallel' fork manager
@@ -35,13 +36,16 @@ class Wmap::UrlCrawler
 		@crawl_depth=params.fetch(:crawl_depth, 4)
 		@crawl_page_limit=params.fetch(:crawl_page_limit, 1000)
 		@max_parallel=params.fetch(:max_parallel, 40)
+		@user_agent=params.fetch(:user_agent, "OWASP WMAP Spider")
 		# Discovered data store
 		@discovered_urls_by_crawler=Hash.new
 		@visited_urls_by_crawler=Hash.new
 		@crawl_start=Hash.new
 		@crawl_done=Hash.new
 		Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
-		@log_file=@data_dir + "/../logs/crawler.log"
+		@log_dir=@data_dir + "/logs/"
+		Dir.mkdir(@log_dir) unless Dir.exist?(@log_dir)
+		@log_file=@log_dir + "crawler.log"
 	end
 
 	# Pre-crawl profiler, to be used for network profiling to maximum the crawler performance.
@@ -216,14 +220,14 @@ class Wmap::UrlCrawler
 	alias_method :crawl_file, :crawl_workers_on_file
 
   # Wrapper for the OpenURI open method - create an open_uri object and return the reference upon success
-	def open_url(url)
+	def open_url(url,user_agent=@user_agent)
 		puts "Open url #{url} by creating an open_uri object. Return the reference upon success." if @verbose
 		if url =~ /http\:/i
 			# patch for allow the 'un-safe' URL redirection i.e. https://www.example.com -> http://www.example.com
-			url_object = open(url, :allow_redirections=>:safe, :read_timeout=>Max_http_timeout/1000)
+			url_object = open(url, :allow_redirections=>:safe, :read_timeout=>Max_http_timeout/1000, "User-Agent"=>user_agent)
 			#url_object = open(url)
 		elsif url =~ /https\:/i
-			url_object = open(url,:ssl_verify_mode => 0, :allow_redirections =>:safe, :read_timeout=>Max_http_timeout/1000)
+			url_object = open(url, :ssl_verify_mode=>0, :allow_redirections=>:safe, :read_timeout=>Max_http_timeout/1000, "User-Agent"=>user_agent)
 			#url_object = open(url,:ssl_verify_mode => 0)
 		else
 			raise "Invalid URL format - please specify the protocol prefix http(s) in the URL: #{url}"
@@ -258,22 +262,6 @@ class Wmap::UrlCrawler
     return nil
   end
 
-=begin
-    # Wrapper for the Nokogiri DOM parser
-	def parse_html(html_body)
-		begin
-			#puts "Parsing the html content: #{html_body}. Return DOM " if @verbose
-      doc = Nokogiri::HTML(html_body)
-			#puts "Successfully crawling the url: #{url_object.base_uri.to_s}" if @verbose
-			#puts "doc: #{doc}" if @verbose
-			return doc
-    rescue => ee
-      puts "Exception on method #{__method__}: #{ee}" if @verbose
-      return nil
-    end
-	end
-=end
-
   # Search 'current_url' and return found URLs under the same domain
 	def find_urls_on_page(doc, current_url)
 		puts "Search and return URLs within the doc: #{doc}" if @verbose

data/lib/wmap/utils/domain_root.rb

@@ -192,7 +192,7 @@ module Wmap
 	# Function to print instance variable - General top level domain list
 	def print_gtld
 		puts @gtld
-    return @gtld 
+    return @gtld
 	end
 
 	# Function to print instance variable - Country code top-level domain list

data/lib/wmap/utils/logger.rb

@@ -8,46 +8,43 @@
 
 
 module Wmap
- module Utils	
+ module Utils
   # Module to log debugging and other messages
-  module Logger	
+  module Logger
 	extend self
 	# Append information into the log file for the trouble-shooting purpose
 	def wlog (obj, agent, file)
 		puts "Writing #{obj} into log file: #{file}" if @verbose
-		begin
-			return false if obj.nil? 
-			# 01/27/2015, implementing singleton pattern for the logger
-			@@f=File.open(file,'a')	
-			timestamp=Time.now
-			case obj
-			when Array
-				if obj.size >= 0
-					@@f.write "#{timestamp}: #{agent}: \n"
-					obj.map { |x| @@f.write "  #{x}\n" }
-					puts "The list is successfully saved into the log file: #{file} " if @verbose
-				end
-			when Hash
-				if obj.length >= 0
-					@@f.write "#{timestamp}: #{agent}: \n"
-					obj.each_value { |value| @@f.write "  #{value}\n" }
-					puts "The hash is successfully saved into the log file: #{file} " if @verbose
-				end
-			when String
-				@@f.write "#{timestamp}: #{agent}: #{obj}\n"
-				puts "The string is successfully saved into the log file: #{file} " if @verbose
-			else
-				#do nothing
-				puts "Un-handled exception on: #{obj}" if @verbose
+		return false if obj.nil?
+		@@f=File.open(file,'a')
+		timestamp=Time.now
+		case obj
+		when Array
+			if obj.size >= 0
+				@@f.write "#{timestamp}: #{agent}: \n"
+				obj.map { |x| @@f.write "  #{x}\n" }
+				puts "The list is successfully saved into the log file: #{file} " if @verbose
 			end
-			@@f.close
-			return true
-		rescue => ee
-			puts "Exception on method #{__method__}: #{ee}" if @verbose
-			return false
-		end	
+		when Hash
+			if obj.length >= 0
+				@@f.write "#{timestamp}: #{agent}: \n"
+				obj.each_value { |value| @@f.write "  #{value}\n" }
+				puts "The hash is successfully saved into the log file: #{file} " if @verbose
+			end
+		when String
+			@@f.write "#{timestamp}: #{agent}: #{obj}\n"
+			puts "The string is successfully saved into the log file: #{file} " if @verbose
+		else
+			#do nothing
+			puts "Un-handled exception on: #{obj}" if @verbose
+		end
+		@@f.close
+		return true
+	rescue => ee
+		puts "Exception on method #{__method__}: #{ee}" if @verbose
+		return false
 	end
-	
-  end	
+
+  end
  end
 end

data/lib/wmap/utils/url_magic.rb

@@ -15,6 +15,7 @@ module Wmap
 
   # set hard stop limit of http time-out to 8 seconds, in order to avoid severe performance penalty for certain 'weird' site(s)
   Max_http_timeout=15000
+  User_agent = "OWASP WMAP Spider"
 
 	# Simple sanity check on a 'claimed' URL string.
 	def is_url?(url)
@@ -263,7 +264,7 @@ module Wmap
 		return absolute_url
   rescue => ee
 		puts "Exception on method #{__method__}: #{ee}" if @verbose
-          return nil
+    return nil
   end
 
 	# Normalize the URL to a consistent manner in order to determine if a link has been visited or cached before
@@ -292,7 +293,6 @@ module Wmap
 		return url
 	end
 
-
 	# Test the URL and return the response code
 	def response_code (url)
 		puts "Check the http response code on the url: #{url}" if @verbose
@@ -344,9 +344,42 @@ module Wmap
 		return code
 	end
 
+	# Test the URL and return the response headers
+	def response_headers (url)
+		puts "Check the http response headers on the url: #{url}" if @verbose
+		raise "Invalid url: #{url}" unless is_url?(url)
+    headers = Hash.new
+		url=url.strip.downcase
+		timeo = Max_http_timeout/1000.0
+		uri = URI.parse(url)
+		http = Net::HTTP.new(uri.host, uri.port)
+		http.open_timeout = timeo
+		http.read_timeout = timeo
+		if (url =~ /https\:/i)
+			http.use_ssl = true
+			#http.ssl_version = :SSLv3
+			# Bypass the remote web server cert validation test
+			http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+		end
+		request = Net::HTTP::Get.new(uri.request_uri)
+		response = http.request(request)
+		puts "Server response the following: #{response}" if @verbose
+		response.each_header do |key,val|
+      puts "#{key} => #{val}" if @verbose
+      headers.merge!({key => val})
+    end
+		puts "Response headers on #{url}: #{headers}" if @verbose
+		return headers
+  rescue => ee
+		puts "Exception on method #{__method__}: #{ee}" if @verbose
+    return nil
+  end
+
+
   # Given an URL, open the page, then return the DOM text from a normal user perspective
   def open_page(url)
-    args = {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE, allow_redirections: :safe, read_timeout: Max_http_timeout/1000}
+    args = {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE, allow_redirections: :safe, \
+      read_timeout: Max_http_timeout/1000, "User-Agent"=>User_agent}
     doc = Nokogiri::HTML(open(url, args))
     if doc.text.include?("Please enable JavaScript to view the page content")
       puts "Invoke headless chrome through webdriver ..." if @verbose
@@ -354,7 +387,7 @@ module Wmap
       #driver = Selenium::WebDriver.for :chrome
       # http://watir.com/guides/chrome/
       args = ['--ignore-certificate-errors', '--disable-popup-blocking', '--disable-translate', '--disk-cache-size 8192']
-      browser = Watir::Browser.new :chrome, headless: true, options: {args: args}
+      browser = Watir::Browser.new :chrome, headless: true, switches: %w[--user-agent=OWASP\ WMAP\ Spider]
       browser.goto(url)
       sleep(2) # wait for the loading
       doc = Nokogiri::HTML(browser.html)

data/lib/wmap/utils/utils.rb

@@ -159,20 +159,18 @@ module Wmap
 	# Simple test a host string format. Return true if it contains a valid internet domain sub-string. Note: Don't be confused with another method 'valid_dns_record?', which is a stricter and time-consuming test on the DNS server for a resolvable internet host.
 	def is_fqdn? (host)
 		puts "Validate the host-name format is valid: #{host}" if @verbose
-		begin
-			return false if is_ip?(host) or is_url?(host)
-			domain=get_domain_root(host)
-			if domain.nil?
-				return false
-			elsif is_domain_root?(domain)
-				return true
-			else
-				return false
-			end
-		rescue => ee
-			puts "Exception on method is_fqdn? for #{host}: #{ee}" if @verbose
+		return false if is_ip?(host) or is_url?(host)
+		domain=get_domain_root(host)
+		if domain.nil?
+			return false
+		elsif is_domain_root?(domain)
+			return true
+		else
 			return false
 		end
+#	rescue => ee
+#		puts "Exception on method is_fqdn? for #{host}: #{ee}" if @verbose
+#		return false
 	end
 	alias_method :is_host?, :is_fqdn?
 

data/lib/wmap/utils/wp_detect.rb

@@ -0,0 +1,358 @@
+#--
+# Wmap
+#
+# A pure Ruby library for Internet web application discovery and tracking.
+#
+# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
+#++
+
+# Utilities for wp_tracker class only; must use with other Utils modules.
+module Wmap
+ module Utils
+    module WpDetect
+	  extend self
+
+    # Main method to detect if it's a wordpress site
+    def is_wp?(url)
+  		site=url_2_site(url)
+  		if wp_readme?(site)
+  			return true
+  		elsif wp_css?(site)
+  			return true
+  		elsif wp_meta?(site)
+  			return true
+  		elsif wp_login?(site)
+  			return true
+  		elsif wp_rpc?(site)
+  			return true
+  		elsif wp_gen?(site)
+  			return true
+      elsif wp_load_styles?(site)
+  			return true
+  		else
+  			return false
+  		end
+  	rescue => ee
+  		puts "Exception on method #{__method__}: #{ee}: #{url}" if @verbose
+  	end
+
+    # Main method to extract the WordPress version
+  	def wp_ver(url)
+  		if !wp_ver_readme(url).nil?
+  			puts "WordPress version found by wp_ver_readme method. " if @verbose
+  			return wp_ver_readme(url)
+  		elsif !wp_ver_login(url,"login.min.css").nil?
+  			puts "WordPress version found by login.min.css file. " if @verbose
+  			return wp_ver_login(url,"login.min.css")
+  		elsif !wp_ver_login(url,"buttons.min.css").nil?
+  			puts "WordPress version found by buttons.min.css file. " if @verbose
+  			return wp_ver_login(url,"buttons.min.css")
+  		elsif !wp_ver_login(url,"wp-admin.min.css").nil?
+  			puts "WordPress version found by wp-admin.min.css file. " if @verbose
+  			return wp_ver_login(url,"wp-admin.min.css")
+  		elsif !wp_ver_meta(url).nil?
+  			puts "WordPress version found by wp_ver_meta method. " if @verbose
+  			return wp_ver_meta(url)
+  		elsif !wp_ver_generator(url).nil?
+  			puts "WordPress version found by wp_ver_generator method. " if @verbose
+  			return wp_ver_generator(url)
+      elsif !wp_ver_load_styles(url).nil?
+  			puts "WordPress version found by wp_ver_load_styles method. " if @verbose
+  			return wp_ver_load_styles(url)
+      else
+  			return nil
+  		end
+  	rescue => ee
+  		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
+  		return nil
+  	end
+
+    # Wordpress detection checkpoint - readme.html
+    def wp_readme?(url)
+  		site = url_2_site(url)
+      readme_url=site + "readme.html"
+      k=Wmap::UrlChecker.new
+      if k.response_code(readme_url) == 200
+        k=nil
+        doc=open_page(readme_url)
+        title=doc.css('title')
+        if title.to_s =~ /wordpress/i
+          return true
+        else
+          return false
+        end
+      else
+        k=nil
+        return false
+      end
+  	rescue => ee
+  		puts "Exception on method #{__method__} for site #{url}: #{ee}" if @verbose
+  		return false
+    end
+
+    # Wordpress detection checkpoint - install.css
+    def wp_css?(url)
+  		site = url_2_site(url)
+      css_url = site + "wp-admin/css/install.css"
+      k=Wmap::UrlChecker.new
+      if k.response_code(css_url) == 200
+        k=nil
+        parser = CssParser::Parser.new
+        parser.load_uri!(css_url)
+        rule = parser.find_by_selector('#logo a')
+        if rule.length >0
+          if rule[0] =~ /wordpress/i
+            return true
+          end
+        end
+      else
+        k=nil
+        return false
+      end
+      return false
+  	rescue => ee
+  		puts "Exception on method #{__method__} for site #{url}: #{ee}" if @verbose
+  		return false
+    end
+
+    # Wordpress detection checkpoint - WP meta tag
+    def wp_meta?(url)
+  		site=url_2_site(url)
+      k=Wmap::UrlChecker.new
+      if k.response_code(site) == 200
+        k=nil
+        doc=open_page(site)
+        meta=doc.css('meta')
+        if meta.to_s =~ /wordpress/i
+          return true
+        else
+          return false
+        end
+      end
+  		return false
+  	rescue => ee
+  		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
+  		return false
+    end
+
+  	# Wordpress detection checkpoint - WP generator tag
+    def wp_gen?(url)
+  		puts "#{__method__} check for #{url}" if @verbose
+  		site = url_2_site(url)
+  		gen_url_1 = site + "feed/"
+  		gen_url_2 = site + "comments/feed"
+      k=Wmap::UrlChecker.new
+      if k.response_code(gen_url_1) == 200
+        doc=open_page(gen_url_1)
+  		elsif k.response_code(gen_url_2) == 200
+  			doc=open_page(gen_url_2)
+  		else
+  			k=nil
+  			return false
+  		end
+  		#puts doc.inspect
+      gens=doc.css('generator')
+  		if gens.nil?
+  			k=nil
+  			return false
+  		end
+  		gens.each do |gen|
+  			if gen.text.to_s =~ /wordpress/i
+  				k=doc=nil
+          return true
+        end
+  		end
+  		k=doc=nil
+  		return false
+  	rescue => ee
+  		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
+  		return false
+    end
+
+  	# Wordpress detection checkpoint - wp-login
+    def wp_login?(url)
+  		site=url_2_site(url)
+  		login_url=site + "wp-login.php"
+      k=Wmap::UrlChecker.new
+      if k.response_code(login_url) == 200
+        k=nil
+        doc=open_page(login_url)
+        links=doc.css('link')
+        if links.to_s =~ /login.min.css/i
+          return true
+        else
+          return false
+        end
+      end
+  		return false
+  	rescue => ee
+  		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
+  		return false
+    end
+
+  	# Wordpress detection checkpoint - xml-rpc
+    def wp_rpc?(url)
+  		site=url_2_site(url)
+  		rpc_url=site + "xmlrpc.php"
+      k=Wmap::UrlChecker.new
+  		#puts "res code", k.response_code(rpc_url)
+      if k.response_code(rpc_url) == 405 # method not allowed
+        k=nil
+        return true
+      end
+  		return false
+  	rescue => ee
+  		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
+  		return false
+    end
+
+    # Wordpress detection checkpoint - /wp-admin/load-styles.php
+    def wp_load_styles?(url)
+  		site = url_2_site(url)
+      load_styles_url=site + "wp-admin/load-styles.php"
+      k=Wmap::UrlChecker.new
+      if k.response_code(load_styles_url) == 200 && k.response_headers(load_styles_url).keys.include?("etag")
+        k=nil
+        return true
+      else
+        k=nil
+        return false
+      end
+  	rescue => ee
+  		puts "Exception on method #{__method__} for site #{url}: #{ee}" if @verbose
+  		return false
+    end
+
+  	# Identify wordpress version through the login page
+    def wp_ver_login(url,pattern)
+  		puts "Check for #{pattern}" if @verbose
+  		site=url_2_site(url)
+  		login_url=site + "wp-login.php"
+      k=Wmap::UrlChecker.new
+  		#puts "Res code: #{k.response_code(login_url)}" if @verbose
+      if k.response_code(login_url) == 200
+        doc=open_page(login_url)
+  			#puts doc.inspect
+        links=doc.css('link')
+  			#puts links.inspect if @verbose
+  			links.each do |tag|
+  	      if tag.to_s.include?(pattern)
+  					puts tag.to_s if @verbose
+  					k=nil
+            if tag.to_s.scan(/[\d+\.]+\d+/).first =~ /\d+\./
+  	           return tag.to_s.scan(/[\d+\.]+\d+/).first
+             else
+               return nil
+             end
+  	      end
+  			end
+      end
+      k=nil
+      return nil
+  	rescue => ee
+  		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
+  		return nil
+    end
+
+  	# Identify wordpress version through the meta link
+    def wp_ver_meta(url)
+  		site=url_2_site(url)
+      k=Wmap::UrlChecker.new
+      if k.response_code(site) == 200
+        doc=open_page(site)
+  			#puts doc.inspect
+        meta=doc.css('meta')
+  			#puts meta.inspect
+  			meta.each do |tag|
+  	      if tag['content'].to_s =~ /wordpress/i
+  					#puts tag.to_s
+  					k=nil
+  	        return tag['content'].to_s.scan(/[\d+\.]+\d+/).first
+  	      end
+  			end
+      end
+      k=nil
+      return nil
+  	rescue => ee
+  		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
+  		return nil
+    end
+
+  	# Identify wordpress version through the generator tag: <generator>https://wordpress.org/?v=4.9.8</generator>
+    def wp_ver_generator(url)
+  		puts "#{__method__} check for #{url}" if @verbose
+  		site = url_2_site(url)
+  		gen_url_1 = site + "feed/"
+  		gen_url_2 = site + "comments/feed"
+      k=Wmap::UrlChecker.new
+      if k.response_code(gen_url_1) == 200
+        doc=open_page(gen_url_1)
+  		elsif k.response_code(gen_url_2) == 200
+  			doc=open_page(gen_url_2)
+  		else
+  			k=nil
+  			return nil
+  		end
+  		#puts doc.inspect
+      gens=doc.css('generator')
+  		if gens.nil?
+  			k=nil
+  			return nil
+  		end
+  		gens.each do |gen|
+  			if gen.text.to_s =~ /wordpress/i
+  				k=nil
+          return gen.text.to_s.scan(/[\d+\.]+\d+/).first
+        end
+  		end
+      k=doc=nil
+      return nil
+  	rescue => ee
+  		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
+  		return nil
+  	end
+
+  	# Wordpress version detection via - readme.html
+    def wp_ver_readme(url)
+  		site=url_2_site(url)
+      readme_url=site + "readme.html"
+      k=Wmap::UrlChecker.new
+  		puts "Res code: #{k.response_code(readme_url)}" if @verbose
+      if k.response_code(readme_url) == 200
+        k=nil
+        doc=open_page(readme_url)
+  			puts doc if @verbose
+        logo=doc.css('h1#logo')[0]
+        puts logo.inspect if @verbose
+  			return logo.to_s.scan(/[\d+\.]+\d+/).first
+      end
+      k=nil
+      return nil
+  	rescue => ee
+  		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
+  		return nil
+  	end
+
+    # Wordpress version detection via - /wp-admin/load-styles.php
+    def wp_ver_load_styles(url)
+  		site=url_2_site(url)
+      load_styles_url = site + "wp-admin/load-styles.php"
+      k=Wmap::UrlChecker.new
+      if k.response_code(load_styles_url) == 200
+        headers = k.response_headers(load_styles_url)
+        if headers.keys.include?("etag")
+          k=nil
+          return headers["etag"]
+        end
+      end
+      k=nil
+      return nil
+  	rescue => ee
+  		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
+  		return nil
+  	end
+
+
+  end
+end
+end