wmap 2.5.2 → 2.5.4

data/lib/wmap/url_crawler/adware_tag.rb

@@ -31,6 +31,7 @@ module Wmap
       File.write(file2, "") unless File.exist?(@tag_file)
       # load the known tag store
       @tag_store=load_tag_from_file(file2)
+      @landings = Hash.new  # cache landing page to reduce redundant browsing
 		end
 
 
@@ -62,7 +63,7 @@ module Wmap
   	end
 
     # load the known tag store cache into an instance variable
-  	def load_tag_from_file (file, lc=true)
+  	def load_tag_from_file (file, lc=false)
       puts "Loading tag data file: #{file}"	if @verbose
 			data_store=Hash.new
 			f = File.open(file, 'r')
@@ -107,8 +108,8 @@ module Wmap
     # add tag entries (from the sitetracker list)
   	def refresh (num=@max_parallel,use_cache=true)
 		  puts "Add entries to the local cache table from site tracker: " if @verbose
-			results=Hash.new
-			tags=Wmap::SiteTracker.instance.known_sites.keys
+			results = Hash.new
+			tags = Wmap::SiteTracker.instance.known_sites.keys
 			if tags.size > 0
 				Parallel.map(tags, :in_processes => num) { |target|
 					check_adware(target,use_cache)
@@ -121,12 +122,12 @@ module Wmap
 				end
 				@tag_store.merge!(results)
 				puts "Done loading entries."
-        tags=nil
+        tags = nil
 				return results
 			else
 				puts "Error: no entry is loaded. Please check your list and try again."
 			end
-      tags=nil
+      tags = nil
 			return results
 		rescue => ee
 			puts "Exception on method #{__method__}: #{ee}" if @verbose
@@ -140,6 +141,10 @@ module Wmap
 			  puts "Site entry already exist. Skipping: #{site}" if @verbose
 			else
         url = fast_landing(site)
+        if @landings.key?(url)
+          record[site] = @landings[url]
+          return record
+        end
         tags = find_tags(url)
         return record if tags.size==0
         tag_vers=tags.map do |tag|
@@ -149,7 +154,8 @@ module Wmap
           Base64.urlsafe_encode64(get_desc(url,tag))
         end
 				if tags
-          record[site]=[url, tags.join("|"), tag_vers.join("|"), tag_descs.join("|")]
+          record[site] = [url, tags.join("|"), tag_vers.join("|"), tag_descs.join("|")]
+          @landings[url] = [url, tags.join("|"), tag_vers.join("|"), tag_descs.join("|")]
           @tag_store.merge!(record)
           puts "Tag entry loaded: #{record}" if @verbose
         else
@@ -189,14 +195,13 @@ module Wmap
   	def find_tags(url)
 			puts "Search and return tags within the url payload: #{url}" if @verbose
 			tag_list = []
-      doc = Nokogiri::HTML(open(url))
+      doc = open_page(url)
       doc.text.each_line do |line|
         my_line = line.downcase
         @tag_signatures.keys.map do |tag|
           tag_list.push(tag) if my_line.include?(tag)
         end
       end
-      doc = nil
       return tag_list
     rescue => ee
       puts "Exception on method #{__method__}: #{ee}" if @verbose
@@ -207,7 +212,7 @@ module Wmap
   	def get_ver(url,tag)
       puts "Search and return tag version within the url payload: #{url}, #{tag}" if @verbose
       tag_ver=""
-      doc = Nokogiri::HTML(open(url))
+      doc = open_page(url)
       case tag
       when "utag.js"          # sample: ...,"code_release_version":"cb20190312032612",...
         doc.text.each_line do |line|
@@ -219,16 +224,33 @@ module Wmap
             break
           end
         end
-      when "analytics.js"          # sample:   ga('create', 'UA-19175804-2', 'knopfdoubleday.com');
+      when "analytics.js"         # sample #1:   ga('create', 'UA-19175804-2', 'knopfdoubleday.com');
         doc.text.each_line do |line|
           my_line = line.downcase
-          if my_line.include?("ga(") && my_line.include?("create")
+          if my_line.include?("ga") && my_line.include?("create")   #sample #2: __gaTracker('create', 'UA-121313929-1', 'auto');
             puts "Extract tag version from line: #{my_line}" if @verbose
             m = my_line.match(/[\'|\"]create[\'|\"]\s*\,\s*[\'|\"](?<ver>\w+\-\d+\-\d+)[\'|\"]\s*\,/)
             tag_ver = m[:ver]
             break
           end
         end
+      when "ga.js"
+        doc.text.each_line do |line|
+          my_line = line.downcase
+          puts my_line if @verbose
+          if my_line.include?("push") && my_line.include?("_setaccount")  # # sample #1:   _gaq.push(['_setAccount', 'UA-13205363-65']);
+            m = my_line.match(/[\'|\"]\_setaccount[\'|\"]\s*\,\s*[\'|\"](?<ver>\w+\-\d+\-\d+)[\'|\"]/)
+            tag_ver = m[:ver]
+            break
+          end
+          if my_line.include?("_gettracker")  # sample #2: var pageTracker = _gat._getTracker("UA-12487327-1");
+            puts "Extract tag version from line: #{my_line}" if @verbose
+            m = my_line.match(/\_gettracker\s*\(\s*[\'|\"](?<ver>\w+\-\d+\-\d+)[\'|\"]/)
+            tag_ver = m[:ver]
+            break
+          end
+
+        end
       when "all.js"          # sample:    appId      : '749936668352954',
         doc.text.each_line do |line|
           my_line = line.downcase
@@ -241,11 +263,11 @@ module Wmap
         end
 
       else
-        puts "Unknown Adware Tag: #{tag}"
+        puts "Don't know how to locate Adware Tag version: #{tag}"
         # do nothing
       end
       doc = nil
-      return tag_ver
+      return tag_ver.upcase
     rescue => ee
       puts "Exception on method #{__method__}: #{ee}: #{url} : #{tag}" if @verbose
       return tag_ver
@@ -257,9 +279,9 @@ module Wmap
       recording=false
       tag_found=false
       tag_desc=""
-      doc = Nokogiri::HTML(open(url))
+      doc = open_page(url)
       doc.search('script').map do |script|
-        if script.text.include?(tag)
+        if script.text.include?(tag) && script.text.length < 65535
           return script.text
         end
       end

data/lib/wmap/utils/domain_root.rb

@@ -21,123 +21,148 @@ module Wmap
 	# Main function to retrieve the registered domain ('domain root' from the 'registrant' perspective) from a hostname, for example, "www.telegraph.co.uk" -> "telegraph.co.uk"
 	def get_domain_root (host)
 		puts "Retrieve the root domain for host: #{host}" if @verbose
-		begin
-      # Comnplete Top Level Domain List - loading once
-      @tlds=file_2_hash(File_tld) if @tlds.nil?
-			# Generic Top Level Domain List - loading once
-			@gtld=file_2_hash(File_gtld) if @gtld.nil?
-			# Country code top-level domain list - loading once
-			@cctld=file_2_hash(File_cctld) if @cctld.nil?
-			# Country code second level domain - loading once
-			@ccsld=load_ccsld_from_file(File_ccsld) if @ccsld.nil?
-
-			if host.strip.nil?
-				puts "Error: empty record found. Please check your input and remove any empty line." if @verbose
-				return nil
-			else
-				host=host.downcase.strip
-			end
-			found_tld=false
-			found_cctld=false
-			# search the  top level domain list first
-			root_domain=""
-			dn=host.split(".")
-			if @tlds.key?(dn.last)
-				cc_found=false
-				if @cctld.key?(dn[dn.length-2])
-					cc_found=true
-				end
-				if cc_found
-					root_domain=dn[dn.length-3] + "." + dn[dn.length-2] + "." + dn.last
-				else
-					root_domain=dn[dn.length-2] + "." + dn.last
-				end
-				found_tld=true
-			end
-			# search the country code top level domain list secondly
-			if @cctld.key?(dn.last)
-				found=false
-				# reverse search of general top level domain
-				if @gtld.key?(dn[dn.length-2])
-					found=true
-				end
-				# search country code second level domain list
-				if @ccsld.key?(dn.last)
-					@ccsld[dn.last].each do |v|
-						if ( v =~ /#{dn[dn.length-2]}/i )
-							found=true
-							break
-						end
-					end
-					# 1/8/2015: additional logic to handle invalid ccsld string: reserved gtld string
-					#unless found
-					#	if @gtld.key?(dn[dn.length-2])
-					#		puts "Invalid ccsld: #{dn[dn.length-2]} for host: #{host}"
-					#		return nil
-					#	end
-					#end
-				end
-				if found
-					root_domain=dn[dn.length-3] + "." + dn[dn.length-2] + "." + dn.last
-				else
-					root_domain=dn[dn.length-2] + "." + dn.last
-				end
-				found_cctld=true
-			end
-			unless (found_tld or found_cctld)
-				puts "#{host} - the top level domain is unknown. Please check out your record #{root_domain} " if @verbose
-				return nil
-			else
-				puts "Domain root found: #{root_domain}" if @verbose
-				return root_domain
-			end
-		rescue => ee
-			puts "Exception on method #{__method__}: #{ee}" if @verbose
+		if host.strip.nil?
+			puts "Error: empty record found. Please check your input and remove any empty line." if @verbose
 			return nil
+		else
+			host=host.downcase.strip
 		end
+    # First order - search country code second level domain list
+    root_domain = get_domain_root_by_ccsld(host)
+    if root_domain.nil?
+  		# Second order - search the country code top level domain list
+      root_domain = get_domain_root_by_cctld(host)
+      if root_domain.nil?
+        # Third order - search top level domain list
+        root_domain = get_domain_root_by_tlds(host)
+        if root_domain.nil?
+          # do nothing - no further search
+        else
+          return root_domain
+        end
+      else
+        return root_domain
+      end
+    else
+      return root_domain
+    end
+    puts "#{host} - the top level domain is unknown. Please check out your record #{root_domain} " if @verbose
+    return nil
+	#rescue => ee
+	#	puts "Exception on method #{__method__}: #{ee}" if @verbose
+	#	return nil
 	end
 	alias_method :get_root_domain, :get_domain_root
 	alias_method :root_domain, :get_domain_root
 	alias_method :domain_root, :get_domain_root
 	alias_method :host_2_domain, :get_domain_root
 
+  # get domain root by lookup Country Code Second Level Domain list
+  def get_domain_root_by_ccsld(host)
+    puts "First order search - domain root lookup by Country Code Second Level Domain list ..." if @verbose
+    root_domain = nil
+    dn = host.split(".")
+    # Country code second level domain - loading once
+		@ccsld=load_ccsld_from_file(File_ccsld) if @ccsld.nil?
+    # search country code second level domain list
+    if @ccsld.key?(dn.last)
+      @ccsld[dn.last].each do |v|
+        if ( v =~ /#{dn[dn.length-2]}/i )
+          return dn[dn.length-3] + "." + dn[dn.length-2] + "." + dn.last
+        end
+      end
+    end
+    return root_domain
+  #rescue => ee
+	#	puts "Exception on method #{__method__}: #{ee}" if @verbose
+	#	return nil
+  end
+
+  # get domain root by lookup Country Code Top Level Domain list
+  def get_domain_root_by_cctld(host)
+    puts "Second order search - domain root lookup by Country Code Top Level Domain list ..." if @verbose
+    root_domain = nil
+    dn = host.split(".")
+    # Country code top-level domain list - loading once
+    @cctld=file_2_hash(File_cctld) if @cctld.nil?
+    # Generic Top Level Domain List - loading once
+    @gtld=file_2_hash(File_gtld) if @gtld.nil?
+    # Country code second level domain - loading once
+		@ccsld=load_ccsld_from_file(File_ccsld) if @ccsld.nil?
+    # search the country code top level domain list
+    if @cctld.key?(dn.last)
+      # reverse search of general top level domain
+      if @gtld.key?(dn[dn.length-2])
+        root_domain=dn[dn.length-3] + "." + dn[dn.length-2] + "." + dn.last
+      else
+        root_domain=dn[dn.length-2] + "." + dn.last
+      end
+    end
+    return root_domain
+  #rescue => ee
+	#	puts "Exception on method #{__method__}: #{ee}" if @verbose
+	#	return nil
+  end
+
+  # get domain root by lookup Top Level Domain list
+  def get_domain_root_by_tlds(host)
+    puts "Third order search - domain root lookup by Top Level Domain list ..." if @verbose
+    root_domain = nil
+    dn = host.split(".")
+    # Comnplete Top Level Domain List - loading once
+    @tlds=file_2_hash(File_tld) if @tlds.nil?
+    # Country code top-level domain list - loading once
+    @cctld=file_2_hash(File_cctld) if @cctld.nil?
+    cc_found=false
+    if @tlds.key?(dn.last)
+      if @cctld.key?(dn[dn.length-2])
+        cc_found=true
+      end
+      if cc_found
+        root_domain=dn[dn.length-3] + "." + dn[dn.length-2] + "." + dn.last
+      else
+        root_domain=dn[dn.length-2] + "." + dn.last
+      end
+    end
+    return root_domain
+    #rescue => ee
+  	#	puts "Exception on method #{__method__}: #{ee}" if @verbose
+  	#	return nil
+  end
+
 	# 'setter' to parse and load the known country code second level domain table from the file
 	# data structure example: {"uk" =>["co","plc"],"za"=>["mil","nom","org"]}
 	def load_ccsld_from_file (file_ccsld)
-		begin
-			ccsld=Hash.new
-			puts "Loading known country code second level domain list from file: #{file_ccsld}" if @verbose
-			f=File.open(file_ccsld, 'r:ISO-8859-1:UTF-8')   # transcoded magic bit
-			f.each do |line|
-				next unless line =~ /^\s+\.\w/
-				line=line.chomp.strip.downcase
-				entry=line.split(' ')[0].split('.')
-				if entry.length > 2
-					key=entry.last
-					ccsld[key] = Array.new if not ccsld.key?(key)
-					val=entry[entry.length-2]
-					#puts "Loading country code second level domain table with - Country code: #{key}, Second level domain: #{val}" if @verbose
-					ccsld[key].push(val) unless key.nil?
-				end
+		ccsld=Hash.new
+		puts "Loading known country code second level domain list from file: #{file_ccsld}" if @verbose
+		f=File.open(file_ccsld, 'r:ISO-8859-1:UTF-8')   # transcoded magic bit
+		f.each do |line|
+			next unless line =~ /^\s+\.\w/
+			line=line.chomp.strip.downcase
+			entry=line.split(' ')[0].split('.')
+			if entry.length > 2
+				key=entry.last
+				ccsld[key] = Array.new if not ccsld.key?(key)
+				val=entry[entry.length-2]
+				#puts "Loading country code second level domain table with - Country code: #{key}, Second level domain: #{val}" if @verbose
+				ccsld[key].push(val) unless key.nil?
 			end
-			f.close
-			# Sort the blocks once in descendant order once for better performance
-			return ccsld
-		rescue => ee
-			puts "Exception on method #{__method__}: #{ee}" if @verbose
 		end
+		f.close
+		# Sort the blocks once in descendant order once for better performance
+		return ccsld
+	rescue => ee
+		puts "Exception on method #{__method__}: #{ee}" if @verbose
 	end
 
 	# Test a host string to see if it's a valid Internet root domain
 	def is_domain_root? (domain)
-		puts "Validate the domain name is valid: #{domain}" if @verbose
-		begin
-			domain=domain.strip.downcase
-			return domain == get_domain_root(domain)
-		rescue => ee
-			puts "Exception on method #{__method__} for #{domain}: #{ee}" if @verbose
-			return false
-		end
+	  puts "Validate the domain name is valid: #{domain}" if @verbose
+		domain=domain.strip.downcase
+		return domain == get_domain_root(domain)
+	rescue => ee
+		puts "Exception on method #{__method__} for #{domain}: #{ee}" if @verbose
+		return false
 	end
 	alias_method :is_root_domain?, :is_domain_root?
 	alias_method :is_domain?, :is_domain_root?
@@ -146,39 +171,40 @@ module Wmap
 	# Function to retrieve the sub-domain from a Fully Qualified Domain Name(FQDN), for example, "www.secure.telegraph.co.uk" -> "secure.telegraph.co.uk"
 	def get_sub_domain (host)
 		puts "Retrieve sub-domain from host: #{host}" if @verbose
-		begin
-			subdomain=String.new
-			host=host.strip.downcase
-			domain=get_domain_root(host)
-			record_h=host.split(".")
-			record_d=domain.split(".")
-			if (record_h.length - record_d.length) >= 2
-				subdomain=record_h[record_h.length-record_d.length-1]+"."+domain
-				puts "Sub domain found: #{subdomain}" if @verbose
-				return subdomain
-			else
-				return nil
-			end
-		rescue Exception => ee
-			puts "Exception on method #{__method__} for #{host}: #{ee}" if @verbose
+		subdomain=String.new
+		host=host.strip.downcase
+		domain=get_domain_root(host)
+		record_h=host.split(".")
+		record_d=domain.split(".")
+		if (record_h.length - record_d.length) >= 2
+			subdomain=record_h[record_h.length-record_d.length-1]+"."+domain
+			puts "Sub domain found: #{subdomain}" if @verbose
+			return subdomain
+		else
 			return nil
 		end
+	rescue Exception => ee
+		puts "Exception on method #{__method__} for #{host}: #{ee}" if @verbose
+		return nil
 	end
 	alias_method :get_subdomain, :get_sub_domain
 
 	# Function to print instance variable - General top level domain list
 	def print_gtld
 		puts @gtld
+    return @gtld 
 	end
 
 	# Function to print instance variable - Country code top-level domain list
 	def print_cctld
 		puts @cctld
+    return @cctld
 	end
 
 	# Function to print instance variable - Country code second-level domain list
 	def print_ccsld
 		puts @ccsld
+    return @ccsld
 	end
 
 	private :load_ccsld_from_file

data/lib/wmap/utils/url_magic.rb

@@ -5,13 +5,17 @@
 #
 # Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
 #++
-# require "uri"
+require "watir"
+require "selenium-webdriver"
 
 module Wmap
- module Utils	
-  module UrlMagic	
+ module Utils
+  module UrlMagic
 	extend self
 
+  # set hard stop limit of http time-out to 8 seconds, in order to avoid severe performance penalty for certain 'weird' site(s)
+  Max_http_timeout=8000
+
 	# Simple sanity check on a 'claimed' URL string.
 	def is_url?(url)
 		puts "Validate the URL format is valid: #{url}" if @verbose
@@ -33,7 +37,7 @@ module Wmap
 			return false
 		end
 	end
-	
+
 	# Simple sanity check on a 'claimed' SSL enabled URL string
 	def is_ssl?(url)
 		puts "Validate if SSL is enabled on: #{url}" if @verbose
@@ -49,8 +53,8 @@ module Wmap
 			return false
 		end
 	end
-	alias_method :is_https?, :is_ssl? 
-	
+	alias_method :is_https?, :is_ssl?
+
 	# Simple sanity check on a 'claimed' web site base string.
 	def is_site?(url)
 		puts "Validate the website string format for: #{url}" if @verbose
@@ -61,7 +65,7 @@ module Wmap
 					return true
 				else
 					return false
-				end			
+				end
 			else
 				puts "Unknown site format: #{url}" if @verbose
 				return false
@@ -71,40 +75,17 @@ module Wmap
 			return nil
 		end
 	end
-	
-	# Check if URL is an absolute one
-	#def is_absolute?(url)
-	#	puts "Validate if the url is absolute: #{url}" if @verbose
-	#	begin
-	#		url.strip!
-	#		URI.absolute?(url)
-	#	rescue => ee
-	#		puts "Exception on method #{__method__} for #{url}: #{ee}" if @verbose
-	#		return false
-	#	end
-	#end
-	
-	# Check if URL is relative one
-	#def is_relative?(url)
-	#	begin
-	#		url.strip!
-	#		!is_absolute?(url)
-	#	rescue => ee
-	#		puts "Exception on method #{__method__} for #{url}: #{ee}" if @verbose
-	#		return false
-	#	end
-	#end	
-	
+
 	# Extract the web server host's Fully Qualified Domain Name (FQDN) from the url. For example: "https://login.yahoo.com/email/help" -> "login.yahoo.com"
 	def url_2_host (url)
 		begin
 			url = url.strip.downcase.gsub(/(http:\/\/|https:\/\/)/, "")
 			record1 = url.split('/')
 			if record1[0].nil?
-				puts "Error process url: #{url}" 
+				puts "Error process url: #{url}"
 				return nil
 			else
-				record2 = record1[0].split(':')		
+				record2 = record1[0].split(':')
 				return record2[0]
 			end
 		rescue => ee
@@ -120,8 +101,8 @@ module Wmap
 			ssl = (url =~ /https/i)
 			url = url.downcase.gsub(/(http:\/\/|https:\/\/)/, "")
 			record1 = url.split('/')
-			record2 = record1[0].split(':')	
-			if (record2.length == 2) 
+			record2 = record1[0].split(':')
+			if (record2.length == 2)
 				puts "The service port: #{record2[1]}" if @verbose
 				return record2[1].to_i
 			elsif ssl
@@ -164,13 +145,13 @@ module Wmap
 			unless is_fqdn?(host)
 				case host
 					# "https://letmechoose.barclays.co.uk?source=btorganic/" => "https://letmechoose.barclays.co.uk"
-					when /\?|\#/	
+					when /\?|\#/
 						host=host.split(%r{\?|\#})[0]
 					else
 						#do nothing
 				end
 			end
-			# step 2, put the host:port pair back to the normal site format 
+			# step 2, put the host:port pair back to the normal site format
 			prot="https:" if port==443
 			if port==80 || port==443
 				site=prot+"//"+host+"/"
@@ -180,7 +161,7 @@ module Wmap
 			if site=~ /http/i
 				#puts "Base found: #{site}" if @verbose
 				return site
-			else	
+			else
 				raise "Problem encountered on method url_2_site: Unable to convert #{url}"
 				return nil
 			end
@@ -202,28 +183,26 @@ module Wmap
 		rescue => ee
 			puts "Exception on method #{__method__} for #{url}: #{ee}" if @verbose
 		end
-	
+
 	end
-	
+
 	# Test if the two URLs are both under the same domain: http://login.yahoo.com, http://mail.yahoo.com => true
 	def urls_on_same_domain?(url1, url2)
 		puts "Determine if two URLs under the same domain: #{url1}, #{url2}" if @verbose
-		begin
-			host1=url_2_host(url1)
-			host2=url_2_host(url2)
-			return get_domain_root(host1) == get_domain_root(host2)
-        rescue => ee
-			puts "Error searching the object content: #{ee}" if @verbose
-            return nil
-        end
-    end	
+		host1=url_2_host(url1)
+		host2=url_2_host(url2)
+		return get_domain_root(host1) == get_domain_root(host2)
+  rescue => ee
+		puts "Error searching the object content: #{ee}" if @verbose
+    return nil
+  end
 
 	# Input is host and open port, output is a URL for valid http response code or nil
 	def host_2_url (host,port=80)
 		puts "Perform simple http(s) service detection on host #{host}, port #{port}" if @verbose
 		begin
 			host=host.strip
-			if port.to_i == 80 
+			if port.to_i == 80
 				url_1 = "http://" + host + "/"
 			elsif port.to_i ==443
 				url_1 = "https://" + host + "/"
@@ -232,7 +211,7 @@ module Wmap
 				url_2 = "https://" + host + ":" + port.to_s + "/"
 			end
 			puts "Please ensure your internet connection is active before running this method: #{__method__}" if @verbose
-			checker=Wmap::UrlChecker.new 
+			checker=Wmap::UrlChecker.new
 			if checker.response_code(url_1) != 10000
 				puts "Found URL: #{url_1}" if @verbose
 				return url_1
@@ -247,8 +226,8 @@ module Wmap
 			puts "Exception on method #{__method__}: #{ee}" if @verbose
 			return nil
 		end
-	end	
-	
+	end
+
 	# Convert a relative URL to an absolute one. For example, from URL base 'http://games.yahoo.com/' and file path '/game/the-magic-snowman-flash.html' => 'http://games.yahoo.com/game/the-magic-snowman-flash.html'
 	def make_absolute(base, relative_url)
         puts "Determine and return the absolute URL:\n Base: #{base}, Relative: #{relative_url} " if @verbose
@@ -266,12 +245,12 @@ module Wmap
             return nil
         end
     end
-    
+
 	# Create / construct the absolute URL from a known URL and relative file path. For example, 'http://images.search.yahoo.com/images' + '/search/images?p=raiders' => 'http://images.search.yahoo.com/search/images?p=raiders'
 	def create_absolute_url_from_base(potential_base, relative_url)
         begin
 			#puts "Determine the absolute URL from potential base #{potential_base} and relative URL #{relative_url}" if @verbose
-			naked_base = url_2_site(potential_base).strip.chop        
+			naked_base = url_2_site(potential_base).strip.chop
 			puts "Found absolute URL: #{naked_base+relative_url}" if @verbose
 			return naked_base + relative_url
         rescue => ee
@@ -309,19 +288,19 @@ module Wmap
             return nil
         end
     end
-	
+
 	# Normalize the URL to a consistent manner in order to determine if a link has been visited or cached before
 	# See http://en.wikipedia.org/wiki/URL_normalization for more explanation
 	def normalize_url(url)
 		begin
 			url.strip!
-			# Converting the scheme and host to lower case in the process, i.e. 'HTTP://www.Example.com/' => 'http://www.example.com/'  
+			# Converting the scheme and host to lower case in the process, i.e. 'HTTP://www.Example.com/' => 'http://www.example.com/'
 			# Normalize the base
-			base=url_2_site(url) 
+			base=url_2_site(url)
 			# Case#1, remove the trailing dot after the hostname, i.e, 'http://www.yahoo.com./' => 'http://www.yahoo.com/'
 			base=base.sub(/\.\/$/,'/')
 			# Normalize the relative path, case#1
-			# retrieve the file path and remove the first '/' or '.', 
+			# retrieve the file path and remove the first '/' or '.',
 			# i.e. 'http://www.example.com/mypath' or 'http://www.example.com/./mypath' => 'mypath'
 			path=url_2_path(url).sub(/^(\/|\.)*/,'')
 			# Normalize the relative path, case#2
@@ -337,7 +316,136 @@ module Wmap
 			return url
 		end
 	end
-	 
+
+
+	# Test the URL and return the response code
+	def response_code (url)
+		puts "Check the http response code on the url: #{url}" if @verbose
+		code = 10000	# All unknown url connection exceptions go here
+		raise "Invalid url: #{url}" unless is_url?(url)
+		url=url.strip.downcase
+		timeo = Max_http_timeout/1000.0
+		uri = URI.parse(url)
+		http = Net::HTTP.new(uri.host, uri.port)
+		http.open_timeout = timeo
+		http.read_timeout = timeo
+		if (url =~ /https\:/i)
+			http.use_ssl = true
+			#http.ssl_version = :SSLv3
+			# Bypass the remote web server cert validation test
+			http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+		end
+		request = Net::HTTP::Get.new(uri.request_uri)
+		response = http.request(request)
+		puts "Server response the following: #{response}" if @verbose
+		code = response.code.to_i
+		#response.finish if response.started?()
+		@url_code=Hash.new unless @url_code
+    @url_code[url]=code
+		puts "Response code on #{url}: #{code}" if @verbose
+		return code
+	rescue Exception => ee
+		puts "Exception on method #{__method__} for #{url}: #{ee}" if @verbose
+		case ee
+		# rescue "Connection reset by peer" error type
+		when Errno::ECONNRESET
+			code=104
+		when Errno::ECONNABORTED,Errno::ETIMEDOUT
+			#code=10000
+		when Timeout::Error				# Quick fix
+			if (url =~ /https\:/i)		# try again for ssl timeout session, in case of default :TLSv1 failure
+				http.ssl_version = :SSLv3
+				response = http.request(request)
+				code = response.code.to_i
+				unless code.nil?
+					@ssl_version = http.ssl_version
+				end
+			end
+		else
+			#code=10000
+		end
+    @url_code=Hash.new unless @url_code
+		@url_code[url]=code
+		return code
+	end
+
+  # Given an URL, open the page, then return the DOM text from a normal user perspective
+  def open_page(url)
+    args = {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE, allow_redirections: :safe, read_timeout: Max_http_timeout/1000}
+    doc = Nokogiri::HTML(open(url, args))
+    if doc.text.include?("Please enable JavaScript to view the page content")
+      puts "Invoke headless chrome through webdriver ..." if @verbose
+      #Selenium::WebDriver::Chrome.path = "/usr/local/bin/chromedriver"
+      #driver = Selenium::WebDriver.for :chrome
+      # http://watir.com/guides/chrome/
+      args = ['--ignore-certificate-errors', '--disable-popup-blocking', '--disable-translate']
+      browser = Watir::Browser.new :chrome, headless: true, options: {args: args}
+      browser.goto(url)
+      sleep(2) # wait for the loading
+      doc = Nokogiri::HTML(browser.html)
+      browser.close
+    end
+    puts doc.text if @verbose
+    return doc
+  rescue => ee
+    puts "Exception on method #{__method__} for #{url}: #{ee}"
+    browser.close unless browser.nil?
+    return doc.text
+  end
+
+  # Test the URL / site and return the redirection location (3xx response code only)
+	def redirect_location (url)
+		puts "Test the redirection location for the url: #{url}" if @verbose
+		location=""
+		raise "Invalid url: #{url}" unless is_url?(url)
+		url=url.strip.downcase
+		timeo = Max_http_timeout/1000.0
+		uri = URI.parse(url)
+		code = response_code (url)
+		if code >= 300 && code < 400
+			http = Net::HTTP.new(uri.host, uri.port)
+			http.open_timeout = timeo
+			http.read_timeout = timeo
+			if (url =~ /https\:/i)
+				http.use_ssl = true
+				# Bypass the remote web server cert validation test
+				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+				http.ssl_version = @ssl_version
+			end
+			request = Net::HTTP::Get.new(uri.request_uri)
+			response = http.request(request)
+			puts "Response: #{response}" if @verbose
+			case response
+			when Net::HTTPRedirection then
+				location = response['location']
+			end
+		end
+		return location
+	rescue Exception => ee
+		puts "Exception on method redirect_location for URL #{url}: #{ee}" if @verbose
+		return ""
+	end
+	alias_method :location, :redirect_location
+
+  # Test the URL / Site and return the landing url location (recursive with the depth = 4 )
+	def landing_location (depth=5, url)
+		depth -= 1
+		return url if depth < 1
+		timeo = Max_http_timeout/1000.0
+		uri = URI.parse(url)
+		code = response_code (url)
+		if code >= 300 && code < 400
+			url = redirect_location (url)
+			url = landing_location(depth,url)
+		else
+			return url
+		end
+		return url
+	rescue Exception => ee
+		puts "Exception on method #{__method__} on URL #{url}: #{ee}" if @verbose
+	end
+
+
   end
  end
 end