wmap 2.7.9 → 2.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 723c7f71bbe80edf1e1c2ed5cad74fe045a8d3e6494b6394921cdd4546d562e7
-  data.tar.gz: 072757ff8e19fee784c3387e362ce9feef86bb7869998df86f8c55437dc9b199
+  metadata.gz: 0430d7df339ee9c4f6099e1175b6cdef5ee0a453c2a5340e72e897e92a24ecfd
+  data.tar.gz: bb4aef610f1f71c992a25c41e0bcd1601039a224a250c251a4fc6b64bc2c362c
 SHA512:
-  metadata.gz: f783d31ad51063734b4ba54cc683399196b838d482cff57d9a27643bcb81451653475953aa2ae4ece6e277ec3756e3a3db05e250c2a6be9595fd7df993088088
-  data.tar.gz: 6502c7c0f5b989440b0dd4f849a1665ddf37cb0cb2d90a673ae31e9928148a1bff164111f46696578b0833ef5ca97a376bfc46793d032bc4596986fd57c1c74b
+  metadata.gz: 371d69e9e72145befa5589a79ca7a7ac541c022881fc4a7ebf2c0ccedd3ab92962557487f7195f216427e1cc3f876efcbcae7cd00adf6ea4e55fec72ec94c5fe
+  data.tar.gz: f3e2245e00a680240b8c0b9f41a2074aff46f5b5ae7cd71ace31be71456a7ee0b909f95223b49dc846cd2f40bdd71cb26f815553943b2b1e78e9b2b567a8008c

data/bin/wmap

@@ -29,6 +29,10 @@ parser.parse!
 # print program banner
 puts Wmap.banner
 # print_usage unless options[:target]
+unless options[:target]
+	puts "Usage:   $ wmap -h"
+	exit 1
+end
 
 # Preparing - check out the working logs directory
 if options[:data_dir]

data/lib/wmap/url_crawler.rb

@@ -17,7 +17,8 @@ require "parallel"
 class Wmap::UrlCrawler
 	include Wmap::Utils
 
-	attr_accessor :http_timeout, :crawl_page_limit, :crawl_depth, :max_parallel, :verbose, :data_dir
+	attr_accessor :http_timeout, :crawl_page_limit, :crawl_depth, :max_parallel, \
+				:verbose, :data_dir, :user_agent
 	attr_reader :discovered_urls_by_crawler, :visited_urls_by_crawler, :crawl_start, :crawl_done
 	# Global variable used to store the combined result of all the forked child processes. Note that class variable
 	# would not be able to pass the result due the limitation of IO Pipe communication mechanism used by 'parallel' fork manager
@@ -35,13 +36,16 @@ class Wmap::UrlCrawler
 		@crawl_depth=params.fetch(:crawl_depth, 4)
 		@crawl_page_limit=params.fetch(:crawl_page_limit, 1000)
 		@max_parallel=params.fetch(:max_parallel, 40)
+		@user_agent=params.fetch(:user_agent, "OWASP WMAP Spider")
 		# Discovered data store
 		@discovered_urls_by_crawler=Hash.new
 		@visited_urls_by_crawler=Hash.new
 		@crawl_start=Hash.new
 		@crawl_done=Hash.new
 		Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
-		@log_file=@data_dir + "/../logs/crawler.log"
+		@log_dir=@data_dir + "/../logs/"
+		Dir.mkdir(@log_dir) unless Dir.exist?(@log_dir)
+		@log_file=@log_dir + "crawler.log"
 	end
 
 	# Pre-crawl profiler, to be used for network profiling to maximum the crawler performance.
@@ -216,14 +220,14 @@ class Wmap::UrlCrawler
 	alias_method :crawl_file, :crawl_workers_on_file
 
   # Wrapper for the OpenURI open method - create an open_uri object and return the reference upon success
-	def open_url(url)
+	def open_url(url,user_agent=@user_agent)
 		puts "Open url #{url} by creating an open_uri object. Return the reference upon success." if @verbose
 		if url =~ /http\:/i
 			# patch for allow the 'un-safe' URL redirection i.e. https://www.example.com -> http://www.example.com
-			url_object = open(url, :allow_redirections=>:safe, :read_timeout=>Max_http_timeout/1000)
+			url_object = open(url, :allow_redirections=>:safe, :read_timeout=>Max_http_timeout/1000, "User-Agent"=>user_agent)
 			#url_object = open(url)
 		elsif url =~ /https\:/i
-			url_object = open(url,:ssl_verify_mode => 0, :allow_redirections =>:safe, :read_timeout=>Max_http_timeout/1000)
+			url_object = open(url, :ssl_verify_mode=>0, :allow_redirections=>:safe, :read_timeout=>Max_http_timeout/1000, "User-Agent"=>user_agent)
 			#url_object = open(url,:ssl_verify_mode => 0)
 		else
 			raise "Invalid URL format - please specify the protocol prefix http(s) in the URL: #{url}"
@@ -258,22 +262,6 @@ class Wmap::UrlCrawler
     return nil
   end
 
-=begin
-    # Wrapper for the Nokogiri DOM parser
-	def parse_html(html_body)
-		begin
-			#puts "Parsing the html content: #{html_body}. Return DOM " if @verbose
-      doc = Nokogiri::HTML(html_body)
-			#puts "Successfully crawling the url: #{url_object.base_uri.to_s}" if @verbose
-			#puts "doc: #{doc}" if @verbose
-			return doc
-    rescue => ee
-      puts "Exception on method #{__method__}: #{ee}" if @verbose
-      return nil
-    end
-	end
-=end
-
   # Search 'current_url' and return found URLs under the same domain
 	def find_urls_on_page(doc, current_url)
 		puts "Search and return URLs within the doc: #{doc}" if @verbose

data/lib/wmap/utils/logger.rb

@@ -8,46 +8,43 @@
 
 
 module Wmap
- module Utils	
+ module Utils
   # Module to log debugging and other messages
-  module Logger	
+  module Logger
 	extend self
 	# Append information into the log file for the trouble-shooting purpose
 	def wlog (obj, agent, file)
 		puts "Writing #{obj} into log file: #{file}" if @verbose
-		begin
-			return false if obj.nil? 
-			# 01/27/2015, implementing singleton pattern for the logger
-			@@f=File.open(file,'a')	
-			timestamp=Time.now
-			case obj
-			when Array
-				if obj.size >= 0
-					@@f.write "#{timestamp}: #{agent}: \n"
-					obj.map { |x| @@f.write "  #{x}\n" }
-					puts "The list is successfully saved into the log file: #{file} " if @verbose
-				end
-			when Hash
-				if obj.length >= 0
-					@@f.write "#{timestamp}: #{agent}: \n"
-					obj.each_value { |value| @@f.write "  #{value}\n" }
-					puts "The hash is successfully saved into the log file: #{file} " if @verbose
-				end
-			when String
-				@@f.write "#{timestamp}: #{agent}: #{obj}\n"
-				puts "The string is successfully saved into the log file: #{file} " if @verbose
-			else
-				#do nothing
-				puts "Un-handled exception on: #{obj}" if @verbose
+		return false if obj.nil?
+		@@f=File.open(file,'a')
+		timestamp=Time.now
+		case obj
+		when Array
+			if obj.size >= 0
+				@@f.write "#{timestamp}: #{agent}: \n"
+				obj.map { |x| @@f.write "  #{x}\n" }
+				puts "The list is successfully saved into the log file: #{file} " if @verbose
 			end
-			@@f.close
-			return true
-		rescue => ee
-			puts "Exception on method #{__method__}: #{ee}" if @verbose
-			return false
-		end	
+		when Hash
+			if obj.length >= 0
+				@@f.write "#{timestamp}: #{agent}: \n"
+				obj.each_value { |value| @@f.write "  #{value}\n" }
+				puts "The hash is successfully saved into the log file: #{file} " if @verbose
+			end
+		when String
+			@@f.write "#{timestamp}: #{agent}: #{obj}\n"
+			puts "The string is successfully saved into the log file: #{file} " if @verbose
+		else
+			#do nothing
+			puts "Un-handled exception on: #{obj}" if @verbose
+		end
+		@@f.close
+		return true
+	rescue => ee
+		puts "Exception on method #{__method__}: #{ee}" if @verbose
+		return false
 	end
-	
-  end	
+
+  end
  end
 end

data/lib/wmap/utils/url_magic.rb

@@ -15,6 +15,7 @@ module Wmap
 
   # set hard stop limit of http time-out to 8 seconds, in order to avoid severe performance penalty for certain 'weird' site(s)
   Max_http_timeout=15000
+  User_agent = "OWASP WMAP Spider"
 
 	# Simple sanity check on a 'claimed' URL string.
 	def is_url?(url)
@@ -377,7 +378,8 @@ module Wmap
 
   # Given an URL, open the page, then return the DOM text from a normal user perspective
   def open_page(url)
-    args = {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE, allow_redirections: :safe, read_timeout: Max_http_timeout/1000}
+    args = {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE, allow_redirections: :safe, \
+      read_timeout: Max_http_timeout/1000, "User-Agent"=>User_agent}
     doc = Nokogiri::HTML(open(url, args))
     if doc.text.include?("Please enable JavaScript to view the page content")
       puts "Invoke headless chrome through webdriver ..." if @verbose
@@ -385,7 +387,7 @@ module Wmap
       #driver = Selenium::WebDriver.for :chrome
       # http://watir.com/guides/chrome/
       args = ['--ignore-certificate-errors', '--disable-popup-blocking', '--disable-translate', '--disk-cache-size 8192']
-      browser = Watir::Browser.new :chrome, headless: true, options: {args: args}
+      browser = Watir::Browser.new :chrome, headless: true, switches: %w[--user-agent=OWASP\ WMAP\ Spider]
       browser.goto(url)
       sleep(2) # wait for the loading
       doc = Nokogiri::HTML(browser.html)

data/version.txt

@@ -3,8 +3,8 @@
 ###############################################################################
 package = wmap
 # wmap version 2.0 == web_discovery version 1.5.3
-version = 2.7.9
-date = 2020-03-30
+version = 2.8.1
+date = 2020-05-08
 
 author = Sam (Yang) Li
 email = yang.li@owasp.org

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wmap
 version: !ruby/object:Gem::Version
-  version: 2.7.9
+  version: 2.8.1
 platform: ruby
 authors:
 - Sam (Yang) Li
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-30 00:00:00.000000000 Z
+date: 2020-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dnsruby