wmap 2.4.6 → 2.4.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7fd34fa6e6a86b6d7a4f2eb0d232023832473ed436870ddab082aa9768cb4bc2
-  data.tar.gz: 7a8e8d44eae5cf2a7c51077cfd7dcfa73a49048833758814d35378438c546195
+  metadata.gz: '039be636ec3179df0106265ea90c9ac86a27f34c1291564a266f97d03105c86c'
+  data.tar.gz: 73453c025547726381e0effd66bae71977b1a242952a6b83e6e8ba7c5d81378f
 SHA512:
-  metadata.gz: 80071d5b153195da339981c4abdc175d4b4368870dd85da822ab2b84fdc66b32e0e7fafcb4b99ab909309f2b90bf671dffb1b5da0a282ea35cc01851c4cf2430
-  data.tar.gz: d8fa25c38b2a147863184a6feca7b94e0588798394046d574030d394b34d9c26ddc5e851539b7ec154de5aa6467660b2eb65cc72137c61d01888d72d23af481d
+  metadata.gz: bdfd196feab5fcd300a8b7856ca1731f90e799cb13ae1545a81aa5ababc859131fdcd9f6f522b6945de200e312e661d89e1cd824e0d0f4587a2d8c7c1e49fe9a
+  data.tar.gz: 27a27879fed90a64f9ff71407388b6e8cbc1ad6e6c8d67964cdfed8a03e729f2a725ed69b738c2c6636912256f3e9fb3a32fd74e923d193b7d5ea5ad290bafa4

data/CHANGELOG.md

@@ -39,9 +39,9 @@
 -	02/11/2015	A bug fix on the Wmap::SiteTracker.get_prim_uniq_sites method, where host resolved to multiple IPs could
 -				  cause exception in the rare case.
 -	02/10/2015	Implement the 'singleton' module in the Wmap::HostTracker::PrimaryHost class
--	02/05/2015	Implement singleton pattern on the Wmap::DomainTracker::SubDomain, Wmap::SiteTracker::DeactivatedSite class.  
+-	02/05/2015	Implement singleton pattern on the Wmap::DomainTracker.instance::SubDomain, Wmap::SiteTracker::DeactivatedSite class.  
 -	02/03/2015	Implement singleton pattern on the logger module, re-organize the log file structure and location.
--	01/30/2015	Implement the singleton pattern on the Wmap::SiteTracker, Wmap::DomainTracker class.
+-	01/30/2015	Implement the singleton pattern on the Wmap::SiteTracker, Wmap::DomainTracker.instance class.
 -	01/27/2015	Separate the logger sub-module and implement the singleton pattern on the logger.
 -	01/24/2015	Implement the 'singleton' module in the Wmap::HostTracker class, in order to avoid race condition
 -				  under the parallel engine

data/bin/RHPG

@@ -10,8 +10,10 @@ def print_usage
 	puts "Program to lookup then merge the site details into RHPG asset spreadsheet. \nUsage: RHPG [RHPG.csv]"
 end
 
+# Lookup the site store for a domain; then return the fingger print info of the site
 def site_lookup(domain)
-  tracker=Wmap::SiteTracker.new(:verbose=>false)
+  tracker=Wmap::SiteTracker.instance
+	tracker.verbose=false
   #first order search
   tracker.known_sites.each do |key,val|
     if key.include?(domain.strip.downcase) && key.include?("https")
@@ -30,6 +32,7 @@ def site_lookup(domain)
   return [nil]*9
 end
 
+# look up the wp site data store for a domain; then return the wp finger print info: [is_wp?,wp_ver]
 def wp_site_lookup(domain)
   tracker=Wmap::WpTracker.new(:verbose=>false)
 	# first order
@@ -42,7 +45,7 @@ def wp_site_lookup(domain)
 	end
   # second order
   tracker.known_wp_sites.each do |key,val|
-    if key.include?(domain.strip.downcase) && key.include?("https")
+    if key.include?(domain.strip.downcase) && key.include?("https") && val
       tracker=nil
       return [val,nil]
     end
@@ -58,6 +61,18 @@ def wp_site_lookup(domain)
   return [nil,nil]
 end
 
+# perform the wpscan on a site
+def wpscan(domain)
+	url=site_lookup(domain)[0]
+	return nil if url.nil?
+	if url.include?("https")
+		command="wpscan --disable-tls-checks --ignore-main-redirect --url=" + url + " -o " + domain + ".wpscan"
+	else
+		command="wpscan --ignore-main-redirect --url=" + url + " -o " + domain + ".wpscan"
+	end
+	system(command)
+end
+
 puts Wmap.banner
 print_usage
 
@@ -71,6 +86,13 @@ CSV.open("output.csv", "wb") do |csv|
     my_row=Array.new
     if cnt > 1
       if is_domain?(row[0])
+=begin
+				if row[3] =~ /Keep/i && row[3] != /Redirect/i
+					unless File.exist?(row[0]+".wpscan")
+						wpscan(row[0])
+					end
+				end
+=end
         my_row = row + site_lookup(row[0]) + wp_site_lookup(row[0])
       else
         my_row = row + [nil]*10

data/bin/distrust

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
-# Executable to remove entry from the ring of the trust. I.E. the trusted domain or CIDR 
+# Executable to remove entry from the ring of the trust. I.E. the trusted domain or CIDR
 require "wmap"
 
 def print_usage
@@ -18,7 +18,7 @@ abort "Incorrect program argument! Proper Usage: distrust [domain | netblock]" u
 puts "Start the demonizing ..."
 
 
-if dt.is_domain?(ARGV[0]) 
+if dt.is_domain?(ARGV[0])
 	result=dt.delete(ARGV[0])
 	unless result.nil?
 		dt.save!
@@ -35,4 +35,3 @@ if ct.is_cidr?(ARGV[0].to_s)
 	end
 end
 ct=nil
-

data/bin/trust

@@ -11,7 +11,7 @@ print_usage
 Log_dir=File.dirname(__FILE__)+'/../logs/'
 Wmap.wlog("Execute the command: trust #{ARGV[0]}","trust",Log_dir+"wmap.log")
 
-dt=Wmap::DomainTracker.new
+dt=Wmap::DomainTracker.instance
 ct=Wmap::CidrTracker.new(:verbose=>true)
 abort "Incorrect program argument! Proper usage: trust [domain | netblock]" unless ARGV.length==1 && (dt.is_fqdn?(ARGV[0]) || ct.is_cidr?(ARGV[0]))
 

data/bin/trusts

@@ -11,7 +11,7 @@ print_usage
 Log_dir=File.dirname(__FILE__)+'/../logs/'
 Wmap.wlog("Execute the command: trust #{ARGV[0]}","trust",Log_dir+"wmap.log")
 
-dt=Wmap::DomainTracker.new
+dt=Wmap::DomainTracker.instance.new
 ct=Wmap::CidrTracker.new(:verbose=>true)
 abort "Incorrect program argument! Proper usage: trust [domain | netblock]" unless ARGV.length==1 && (File.exist?(ARGV[0]))
 

data/bin/updateAll

@@ -19,33 +19,40 @@ else
 	abort "You got it. Mission is successfully aborted. "
 end
 # Update sub-domain table
-sd=Wmap::DomainTracker::SubDomain.new
+sd=Wmap::DomainTracker::SubDomain.instance
 sd.update_from_host_store!
 subs=sd.known_internet_sub_domains.keys
 sd=nil
 
+# Update Domain table
+dm=Wmap::DomainTracker.instance
+domains=dm.known_internet_domains.keys
+dm=nil
+
 # Brute force sub-domains to detect sub-domain hosts
 bruter=Wmap::DnsBruter.new
 sub_hosts=bruter.brutes(subs).values.flatten
+hosts=bruter.brutes(domains).values.flatten
 
 # Update primary host store
-ph=Wmap::HostTracker::PrimaryHost.new
+ph=Wmap::HostTracker::PrimaryHost.instance
 ph.update_from_site_store!
 ph.refresh_all
 ph.save!
 ph=nil
 
 # Update host store
-h=Wmap::HostTracker.new
+h=Wmap::HostTracker.instance
 h.refresh_all
 h.adds(sub_hosts)
+h.adds(hosts)
 h.save!
 h=nil
 
 # Update site store
-st=Wmap::SiteTracker.new
+st=Wmap::SiteTracker.instance
 st.refresh_all
-dt=Wmap::SiteTracker::DeactivatedSite.new
+dt=Wmap::SiteTracker::DeactivatedSite.instance
 ds=dt.known_sites.keys
 st.adds(ds)					#double-check the de-activated sites in case the site is back on-line again
 st.save!

data/bin/wadd

@@ -4,16 +4,35 @@
 require "wmap"
 
 def print_usage
-	puts "Program to add an entry into the local data repository. Usage: wadd [site]"
+	puts "Program to add an entry into the local data repository. Usage: wadd <site> [data_dir]"
 end
 
 puts Wmap.banner
 print_usage
-Log_dir=File.dirname(__FILE__)+'/../logs/'
+if ARGV.length == 1
+	# Log the command entry
+	Log_dir = File.dirname(__FILE__) + '/../logs/'
+elsif ARGV.length == 2
+	# Log to the instance running directory
+	Log_dir = File.dirname(__FILE__) + '/../logs/' + ARGV[1]
+else
+	Log_dir = File.dirname(__FILE__) + '/../logs/'
+end
+Dir.mkdir(Log_dir) unless Dir.exist?(Log_dir)
 Wmap.wlog("Execute the command: wadd #{ARGV[0]}","wadd",Log_dir+"wmap.log")
 
-st=Wmap::SiteTracker.instance
-abort "Incorrect program argument!" unless ARGV.length==1
+if ARGV.length == 1
+	puts puts "Invoke the SiteTracker."
+	st=Wmap::SiteTracker.instance
+	st.verbose=false
+elsif ARGV.length == 2
+	puts puts "Invoke the SiteTracker."
+	st=Wmap::SiteTracker.instance
+	st.verbose=false
+	st.data_dir=ARGV[1]
+else
+	aborts "Error firing up SiteTracker instance!"
+end
 
 # Evaluate the argument and update the data store accordingly
 if st.is_site?(ARGV[0])

data/bin/wadds

@@ -8,11 +8,30 @@ end
 
 puts Wmap.banner
 print_usage
-Log_dir=File.dirname(__FILE__)+'/../logs/'
+abort "Incorrect program argument!" unless File.exist?(ARGV[0])
+
+if ARGV.length == 1
+	# Log the command entry
+	Log_dir = File.dirname(__FILE__)+'/../logs/'
+elsif ARGV.length == 2
+	# Log to the instance running directory
+	Log_dir = File.dirname(__FILE__)+'/../logs/' + ARGV[1]
+end
+Dir.mkdir(Log_dir) unless Dir.exist?(Log_dir)
 Wmap.wlog("Execute the command: wadds #{ARGV[0]}","wadds",Log_dir+"wmap.log")
 
-st=Wmap::SiteTracker.new
-abort "Incorrect program argument!" unless ARGV.length==1 && File.exist?(ARGV[0])
+if ARGV.length == 1
+	puts puts "Invoke the SiteTracker."
+	st=Wmap::SiteTracker.instance
+	st.verbose=false
+elsif ARGV.length == 2
+	puts puts "Invoke the SiteTracker."
+	st=Wmap::SiteTracker.instance
+	st.verbose=false
+	st.data_dir=ARGV[1]
+else
+	aborts "Error firing up SiteTracker instance!"
+end
 
 # Evaluate the argument and update the data store accordingly
 sites=st.file_2_list(ARGV[0]).map { |x| st.url_2_site(x) }
@@ -22,5 +41,6 @@ if sites.length > 0
 	st.save! if news.length>0
 	st=nil
 else
+	st=nil
 	abort "No site entry found in file: #{ARGV[0]}. Please check your file format to ensure one site per line."
 end

data/bin/wdel

@@ -4,16 +4,35 @@
 require "wmap"
 
 def print_usage
-	puts "Program to delete an entry from the local data repository. Usage: wdel [site]"
+	puts "Program to delete an entry from the local data repository. Usage: wdel <site> [data_dir]"
 end
 
 puts Wmap.banner
 print_usage
-Log_dir=File.dirname(__FILE__)+'/../logs/'
+if ARGV.length == 1
+	# Log the command entry
+	Log_dir = File.dirname(__FILE__) + '/../logs/'
+elsif ARGV.length == 2
+	# Log to the instance running directory
+	Log_dir = File.dirname(__FILE__) + '/../logs/' + ARGV[1]
+else
+	Log_dir = File.dirname(__FILE__) + '/../logs/'
+end
+Dir.mkdir(Log_dir) unless Dir.exist?(Log_dir)
 Wmap.wlog("Execute the command: wdel #{ARGV[0]}","wdel",Log_dir+"wmap.log")
 
-st=Wmap::SiteTracker.instance
-abort "Incorrect program argument!" unless ARGV.length==1
+if ARGV.length == 1
+	puts puts "Invoke the SiteTracker."
+	st=Wmap::SiteTracker.instance
+	st.verbose=false
+elsif ARGV.length == 2
+	puts puts "Invoke the SiteTracker."
+	st=Wmap::SiteTracker.instance
+	st.verbose=false
+	st.data_dir=ARGV[1]
+else
+	aborts "Error firing up SiteTracker instance!"
+end
 
 # Evaluate the argument and update the data store accordingly
 if st.is_site?(ARGV[0])

data/bin/wmap

@@ -16,9 +16,9 @@ if ARGV.length == 1
 	Log_dir=File.dirname(__FILE__)+'/../logs/'
 elsif ARGV.length == 2
 	# Log to the instance running directory
-	Log_dir=ARGV[1] + '/logs/'
-	Dir.mkdir(Log_dir) unless Dir.exist?(Log_dir)
+	Log_dir=File.dirname(__FILE__)+'/../logs/'+ARGV[1]
 end
+Dir.mkdir(Log_dir) unless Dir.exist?(Log_dir)
 
 Wmap.wlog("Execute the command: wmap #{ARGV[0]}","wmap",Log_dir+"wmap.log")
 print_usage unless (ARGV.length==1 or ARGV.length==2)
@@ -67,10 +67,13 @@ end
 # second step - update the hosts repository
 if ARGV.length == 1
 	puts puts "Invoke the HostTracker."
-	host_tracker = Wmap::HostTracker.new(:verbose=>true)
+	host_tracker = Wmap::HostTracker.instance
+	host_tracker.verbose=true
 elsif ARGV.length == 2
 	puts "Invoke the HostTracker with optional directory setter."
-	host_tracker = Wmap::HostTracker.new(:verbose=>true, :data_dir => ARGV[1])
+	host_tracker = Wmap::HostTracker.instance
+	host_tracker.verbose=true
+	host_tracker.data_dir = ARGV[1]
 else
 	aborts "Error firing up HostTracker instance!"
 end
@@ -137,10 +140,11 @@ else
 	puts "Automatically save the discovery results into the site tracking data repository: "
 	if ARGV.length == 1
 		puts "Start the SiteTracker. "
-		inventory=Wmap::SiteTracker.new
+		inventory=Wmap::SiteTracker.instance
 	elsif ARGV.length == 2
 		puts "Start the SiteTracker with the optional directory setter. "
-		inventory=Wmap::SiteTracker.new(:data_dir => ARGV[1])
+		inventory=Wmap::SiteTracker.instance
+		inventory.data_dir = ARGV[1]
 	else
 		aborts "Error firing up SiteTracker instance!"
 	end

data/bin/wmaps

@@ -10,15 +10,14 @@ def wmap_worker(domain)
 end
 
 
-tracker=Wmap::DomainTracker.new
-=begin
+tracker=Wmap::DomainTracker.instance
 Parallel.map(tracker.known_internet_domains.keys, :in_processes => 10) { |target|
   puts "Working on #{target} ..." if @verbose
   wmap_worker(target)
 }
-=end
+=begin
 tracker.known_internet_domains.keys.map do |domain|
   wmap_worker(domain)
 end
-
+=end
 tracker=nil

data/demos/bruter.rb

@@ -4,7 +4,7 @@ require "wmap"
 
 f_rpt=".rpt.txt"
 # Step 1 - obtain list of domains to be brute-forced on
-host_tracker=Wmap::HostTracker.new
+host_tracker=Wmap::HostTracker.instance
 root_domains=host_tracker.dump_root_domains
 sub_domains=host_tracker.instance.dump_sub_domains
 # Step 2 - multi-thread brute forcer works on known domains and sub-domains

data/demos/dns_brutes.rb

@@ -4,7 +4,7 @@ require "wmap"
 
 f_rpt=".rpt.txt"
 # Step 1 - obtain list of domains to be brute-forced on
-tracker=Wmap::HostTracker.new
+tracker=Wmap::HostTracker.instance
 root_domains=tracker.dump_root_domains
 sub_domains=tracker.dump_sub_domains
 # Step 2 - multi-thread brute forcer works on known domains and sub-domains 

data/demos/filter_domain.rb

@@ -4,7 +4,7 @@ require	"wmap"
 
 puts Wmap.banner
 puts "Host,Domain,IP,Trusted CIDR,Trusted CIDR Netname"
-myD = Wmap::DomainTracker.new
+myD = Wmap::DomainTracker.instance
 f_urls = File.open(ARGV[0], 'r')
 f_urls.each do |line|
 	url=line.chomp

data/demos/filter_known_services.rb

@@ -6,7 +6,7 @@ require "wmap"
 
 # Create a known service map by parsing last quarter scan list
 def parse_old
-	host_tracker=Wmap::HostTracker.new
+	host_tracker=Wmap::HostTracker.instance
 	@services=Hash.new
 	f_site=File.open(ARGV[0],'r')
 	f_site.each do |line|
@@ -30,7 +30,7 @@ end
 
 # Go through the new scan list and look up for known service from last quarter
 def diff
-	host_tracker=Wmap::HostTracker.new
+	host_tracker=Wmap::HostTracker.instance
 	f_new = File.open(ARGV[1],'r')
 	f_new.each do |line|
 		site=line.chomp.strip

data/demos/filter_prime.rb

@@ -3,7 +3,7 @@
 require	"wmap"
 
 puts "URL | Prime URL"
-my_tracker=Wmap::HostTracker::PrimaryHost.new
+my_tracker=Wmap::HostTracker::PrimaryHost.instance
 f_urls = File.open(ARGV[0], 'r')
 f_urls.each do |line|
 	url=line.chomp

data/demos/filter_site.rb

@@ -7,7 +7,7 @@ require "wmap"
 @map=Hash.new
 
 def build_map (file)
-	k=Wmap::SiteTracker.new
+	k=Wmap::SiteTracker.instance
 	f=File.open(file,'r')
 	f.each do |line|
 		url=line.chomp.strip.downcase
@@ -23,7 +23,7 @@ end
 
 
 build_map(ARGV[0])
-s=Wmap::SiteTracker.new
+s=Wmap::SiteTracker.instance
 f=File.open(ARGV[1],'r')
 f.each do |line|
 	url=line.chomp.strip.downcase

data/demos/filter_siteip.rb

@@ -4,7 +4,7 @@ require	"wmap"
 
 def known?(ip)
 	ip=ip.chomp.strip
-	myDis = Wmap::SiteTracker.new
+	myDis = Wmap::SiteTracker.instance
 	known=false
 	if myDis.is_ip?(ip)
 		if myDis.siteip_known?(ip)

data/demos/filter_url.rb

@@ -4,7 +4,7 @@
 
 require "wmap"
 
-k=Wmap::DomainTracker.new
+k=Wmap::DomainTracker.instance.new
 #k.verbose=true
 f=File.open(ARGV[0],'r')
 f.each do |line|

data/demos/new_fnd.rb

@@ -11,7 +11,7 @@ require "wmap"
 
 def load_keys (file)
 	puts "Load the key map from file: #{file}" if @verbose
-	host_tracker=Wmap::HostTracker.new
+	host_tracker=Wmap::HostTracker.instance
 	my_keys=Hash.new
 	#begin
 		f_old=File.open(file)
@@ -45,7 +45,7 @@ def load_keys (file)
 end
 
 old_keys=load_keys(ARGV[0])
-my_tracker=Wmap::HostTracker.new
+my_tracker=Wmap::HostTracker.instance
 f_new=File.open(ARGV[1],'r')
 f_new.each_line do |line|
 	ent=line.chomp.split(',')

data/demos/site_format.rb

@@ -4,7 +4,7 @@
 
 require "wmap"
 
-k=Wmap::SiteTracker.new
+k=Wmap::SiteTracker.instance
 f=File.open(ARGV[0],'r')
 f.each do |line|
 	url=line.chomp.strip.downcase

data/demos/whois_domain.rb

@@ -5,7 +5,7 @@
 require	"wmap"
 
 puts Wmap.banner
-dis=Wmap::DomainTracker.new
+dis=Wmap::DomainTracker.instance
 dis.verbose=false
 puts "Domain Whois Lookup Summary Report"
 puts "Host | Domain | Primary Domain Name Server | Registrant Name | Registrant Oraganization | Registrant Address | Registrant Zip | Registrant City | Registrant State | Registration Country | Registration Contact Phone | Registration Contact Email | Technical Contact Name | Technical Contact Organization | Technical Contact Country | Technical Contract Phone | Technical Contact Email | Admin Contact Name | Admin Contact Organization | Domain Availability"
@@ -75,4 +75,4 @@ f_hosts.each do |line|
 	end
 end
 f_hosts.close
-dis=nil 
+dis=nil

data/lib/wmap/dns_bruter.rb

@@ -163,7 +163,8 @@ class Wmap::DnsBruter
 		begin
 			host=host.strip
 			valid_hosts = Array.new
-			my_host_tracker = Wmap::HostTracker.new(:data_dir=>@data_dir)
+			my_host_tracker = Wmap::HostTracker.instance
+			my_host_tracker.data_dir=@data_dir
 			# build the host dictionary for the brute force method
 			dict = Array.new
 			if File.exists?(@hosts_dict)
@@ -247,7 +248,8 @@ class Wmap::DnsBruter
 		puts "Start the parallel brute-forcing all domains with maximum child processes: #{num}"
 		begin
 			hosts=Array.new
-			my_dis=Wmap::HostTracker.new(:data_dir=>@data_dir)
+			my_dis=Wmap::HostTracker.instance
+			my_dis.data_dir=@data_dir
 			known_domains=my_dis.dump_root_domains
 			hosts=dns_brute_domains(num, known_domains)
 			my_dis.adds(hosts)

data/lib/wmap/domain_tracker/sub_domain.rb

@@ -5,7 +5,7 @@
 #
 # Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
 #++
-#require "singleton"
+require "singleton"
 
 
 module Wmap
@@ -15,7 +15,7 @@ class DomainTracker
 # of the sub-domains and the associated entities
 class SubDomain < Wmap::DomainTracker
 	include Wmap::Utils
-	#include Singleton
+	include Singleton
 
 	attr_accessor :verbose, :domains_file, :max_parallel, :data_dir
 	attr_reader :known_internet_sub_domains
@@ -90,7 +90,10 @@ class SubDomain < Wmap::DomainTracker
 		puts "Invoke internal procedures to update the sub-domain list from the host store."
 		begin
 			# Step 1 - obtain the latest sub-domains
-			subs = Wmap::HostTracker.new(:data_dir=>@data_dir).dump_sub_domains - [nil,""]
+			my_tracker = Wmap::HostTracker.instance
+			my_tracker.data_dir=@data_dir
+			subs = my_tracker.dump_sub_domains - [nil,""]
+			my_tracker = nil 
 			# Step 2 - update the sub-domain list
 			unless subs.empty?
 				#subs.map { |x| self.add(x) unless domain_known?(x) }

data/lib/wmap/domain_tracker.rb

@@ -6,13 +6,13 @@
 # Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
 #++
 require "parallel"
-#require "singleton"
+require "singleton"
 
 
 # Class to track the known (trusted) Internet domains
 class Wmap::DomainTracker
 	include Wmap::Utils
-	#include Singleton
+	include Singleton
 
 
 	attr_accessor :verbose, :max_parallel, :domains_file, :file_domains, :data_dir
@@ -110,6 +110,7 @@ class Wmap::DomainTracker
 	def add(host)
 		puts "Add entry to the local domains cache table: #{host}" if @verbose
 		#begin
+			return nil if host.nil? or host.empty?
 			host=host.strip.downcase
 			if @known_internet_domains.key?(host)
 				puts "Domain is already exist. Skipping: #{host}"

data/lib/wmap/host_tracker/primary_host.rb

@@ -5,7 +5,7 @@
 #
 # Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
 #++
-#require "singleton"		# Implement singleton pattern to avoid race condition under parallel engine
+require "singleton"		# Implement singleton pattern to avoid race condition under parallel engine
 
 
 module Wmap
@@ -14,7 +14,7 @@ module Wmap
 	# Class to differentiate the primary host-name from the potential aliases. This is needed in order to minimize the confusion on our final site inventory list, as it contains a large number of duplicates (aliases). More specifically, a filter could be built by using this class to track the primary url of a website.
 	class PrimaryHost < Wmap::HostTracker
 		include Wmap::Utils
-		#include Singleton
+		include Singleton
 
 		attr_accessor :hosts_file, :verbose, :data_dir
 		attr_reader :known_hosts, :known_ips
@@ -35,12 +35,13 @@ module Wmap
 
 		# Procedures to identify primary host-name from the site store SSL certificates. The assumption is that the CN used in the cert application must be primary hostname and used by the users.
 		def update_from_site_store!
-			puts "Invoke internal procedures to update the primary host-name table from the site store."
-			begin
-				# Step 1 - update the prime host table based on the SSL cert CN fields
+			#begin
+        puts "Invoke internal procedures to update the primary host-name table from the site store."
+        # Step 1 - update the prime host table based on the SSL cert CN fields
 				cns=Hash.new
 				checker=Wmap::UrlChecker.new(:data_dir=>@data_dir)
-        my_tracker = Wmap::SiteTracker.new(:data_dir=>@data_dir)
+        my_tracker = Wmap::SiteTracker.instance
+        my_tracker.data_dir = @data_dir
 				my_tracker.get_ssl_sites.map do |site|
 					puts "Exam SSL enabled site entry #{site} ..."
 					my_host=url_2_host(site)
@@ -62,12 +63,12 @@ module Wmap
 				self.save!
         checker=nil
         my_tracker=nil
-			rescue Exception => ee
-				puts "Exception on method #{__method__}: #{ee}" if @verbose
-        checker=nil
-        my_tracker=nil
-				return nil
-			end
+			#rescue Exception => ee
+			#	puts "Exception on method #{__method__}: #{ee}" if @verbose
+      #  checker=nil
+      #  my_tracker=nil
+			#	return nil
+			#end
 		end
 		alias_method :update!, :update_from_site_store!
 
@@ -75,7 +76,10 @@ module Wmap
 		def update_from_site_redirections!
 			puts "Invoke internal procedures to update the primary host-name table from the site store."
 			begin
-				urls=Wmap::SiteTracker.new(:data_dir=>@data_dir).get_redirection_urls
+				my_tracker=Wmap::SiteTracker.instance
+        my_tracker.data_dir=@data_dir
+        urls = my_tracker.get_redirection_urls
+        my_tracker = nil 
 				urls.map do |url|
 					if is_url?(url)
 						host=url_2_host(url)