wmap 2.8.2 → 2.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2dee2577809daef8231ddc3d97a7e0abab5312700f36ab2233366ff6729ae388
-  data.tar.gz: 5dee805b28f88e2ae320e6afd9b4ef7d6f25bff99a34cf46b42c32249bf7eaad
+  metadata.gz: a6632168c88c35189b601d225ebbf99fddd034a561814e5fe34b2d57bb79c75c
+  data.tar.gz: 4beab7d92e6a5e4258d37dfa52a64f4edaf06d8c4213331cc9833d0be5cc70aa
 SHA512:
-  metadata.gz: c78742dad1d356b88d2b45a0773527fd8f4e54cbff13c0ccb5f9fd9a228fac56cc2c26a705e8c5ce068b4d2bdf50d1fd190060501a1fc17e0a1040e583080db6
-  data.tar.gz: 7eea2eaa6ee45e9ac7857eaa603177b00424a6842cdd89a200062bdcd737c1a66fa2310645e80272fce71d0193fb9ca8044f232e0dabdf20c2da453e436c84cc
+  metadata.gz: 46482f94126bc1ad4af322cb23894e77ee5a0f85dc31741e25b75752be4a47b1a620c3d1a6f28786a2f53e37eecde42b571f011e770e4bacdaf00012d737ea9e
+  data.tar.gz: b5ed0e1af39d8b7910b0d77b5bde99461293a7e985d5e587405bbf929cae8bffa5a32a1cf31eb142f823a11057377537adecf9ee6fb7f475fb2252e9d3edc2b1

data/bin/wmap

@@ -146,22 +146,23 @@ when nil,[]
 	puts "No new site found. There is no change to the site tracking data repository. "
 else
 	puts "Automatically save the discovery results into the site tracking data repository: "
+	inventory = Wmap::SiteTracker.instance
 	if options[:target] && options[:data_dir]
 		puts "Start the SiteTracker with the optional directory setter. "
-		inventory=Wmap::SiteTracker.instance
 		inventory.data_dir = options[:data_dir]
+		#inventory.verbose = true
 		inventory.sites_file = inventory.data_dir + "/" + "sites"
 		inventory.load_site_stores_from_file(inventory.sites_file)
 	elsif options[:target]
 		puts "Start the SiteTracker. "
-		inventory=Wmap::SiteTracker.instance
 	else
 		abort "Error firing up SiteTracker instance!"
 	end
 	new_sites=inventory.adds(dis_sites.keys-["",nil])
-	if new_sites.size>0 && options[:data_dir]
+	puts "Newly discovery sties: #{new_sites}"
+	if options[:data_dir]
 		inventory.save!(inventory.sites_file)
-	elsif new_sites.size>0
+	else
 		inventory.save!
 	end
 	inventory=nil
@@ -170,17 +171,17 @@ end
 
 
 # seventh step - update the hosts repository
+puts "Invoke the HostTracker with optional directory setter."
+host_tracker = Wmap::HostTracker.instance
 if options[:target] && options[:data_dir]
-	puts "Invoke the HostTracker with optional directory setter."
-	host_tracker = Wmap::HostTracker.instance
+	puts puts "Invoke the HostTracker with options: #{options[:data_dir]}, #{options[:target]}"
 	host_tracker.verbose=options[:verbose]
 	host_tracker.data_dir = options[:data_dir]
 	host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
 	host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
 elsif options[:target]
-	puts puts "Invoke the HostTracker."
-	host_tracker = Wmap::HostTracker.instance
-	host_tracker.verbose=options[:verbose]
+	puts puts "Invoke the HostTracker with option: #{options[:target]}."
+	#host_tracker.verbose=options[:verbose]
 else
 	abort "Error firing up HostTracker instance!"
 end

data/lib/wmap/cidr_tracker.rb

@@ -18,7 +18,7 @@ class Wmap::CidrTracker
 		@verbose=params.fetch(:verbose, false)
 		@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
 		Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
-		@cidr_seeds=params.fetch(:cidr_seeds, @data_dir + 'cidrs')
+		@cidr_seeds=params.fetch(:cidr_seeds, @data_dir + '/' + 'cidrs')
 		File.write(@cidr_seeds, "") unless File.exist?(@cidr_seeds)
 		load_cidr_blks_from_file(@cidr_seeds)
 	end

data/lib/wmap/domain_tracker.rb

@@ -169,7 +169,7 @@ class Wmap::DomainTracker
 				end
 			end
 			@known_internet_domains.merge!(results)
-			puts "Done loading entries."
+			puts "Done loading domain entries."
 			return results
 		else
 			puts "Error: no entry is loaded. Please check your list and try again."
@@ -262,7 +262,7 @@ class Wmap::DomainTracker
 		when "Wmap::DomainTracker::SubDomain"
 			return @known_internet_sub_domains.key?(domain)
 		else
-			return nil
+			return false
 		end
 	rescue => ee
 		puts "Exception on method #{__method__}: #{ee}" if @verbose

data/lib/wmap/domain_tracker/sub_domain.rb

@@ -70,7 +70,7 @@ class SubDomain < Wmap::DomainTracker
 				end
 			end
 			@known_internet_sub_domains.merge!(results)
-			puts "Done loading entries."
+			puts "Done loading sub_domain entries."
 			return results
 		else
 			puts "Error: no entry is loaded. Please check your list and try again."

data/lib/wmap/host_tracker.rb

@@ -27,13 +27,13 @@ class Wmap::HostTracker
 		@max_parallel=params.fetch(:max_parallel, 40)
 		# Initialize the instance variables
 		File.write(@hosts_file, "") unless File.exist?(@hosts_file)
-		load_known_hosts_from_file(@hosts_file)
+		@known_hosts=load_known_hosts_from_file(@hosts_file)
 	end
 
 	# Setter to load the known hosts from the local hosts file into a class instance
 	def load_known_hosts_from_file (f_hosts=@hosts_file)
 		puts "Loading local hosts from file: #{f_hosts} ..." if @verbose
-		@known_hosts=Hash.new
+		known_hosts=Hash.new
 		@alias = Hash.new
 		File.write(f_hosts, "") unless File.exist?(f_hosts)
 		f=File.open(f_hosts, 'r')
@@ -43,11 +43,11 @@ class Wmap::HostTracker
 			key=entry[0].downcase
 			value=entry[1]
 			puts "Loading key value pair: #{key} - #{value}" if @verbose
-			@known_hosts[key] = Hash.new unless @known_hosts.key?(key)
-			@known_hosts[key]= value
+			known_hosts[key] = Hash.new unless known_hosts.key?(key)
+			known_hosts[key]= value
 			# For reverse host lookup
-			@known_hosts[value] = Hash.new unless @known_hosts.key?(value)
-			@known_hosts[value] = key
+			known_hosts[value] = Hash.new unless known_hosts.key?(value)
+			known_hosts[value] = key
 			# Count the number of alias for the recorded IP
 			if @alias.key?(value)
 				@alias[value]+=1
@@ -56,11 +56,12 @@ class Wmap::HostTracker
 			end
 		end
 		f.close
-		return @known_hosts
-	rescue => ee
-		puts "Exception on method #{__method__}: #{ee}"
 		return known_hosts
+		#rescue => ee
+		#	puts "Exception on method #{__method__}: #{ee}"
+		#	return known_hosts
 	end
+	alias_method :load, :load_known_hosts_from_file
 
 	# Save the current local hosts hash table into a (random) data repository file
 	def save_known_hosts_to_file!(f_hosts=@hosts_file)
@@ -96,30 +97,42 @@ class Wmap::HostTracker
 		puts "Exception on method #{__method__}: #{ee}"
 	end
 
+  # determine if host is part of trusted (known) root domains
+	def is_trusted?(host)
+		puts "Determin if host #{host} is part of trusted root domains" if @verbose
+		root=get_domain_root(host)
+		puts "Domain root: #{root}" if @verbose
+		domain_tracker=Wmap::DomainTracker.instance
+		domain_tracker.data_dir=@data_dir
+		domain_tracker.domains_file = domain_tracker.data_dir + "/" + "domains"
+		domain_tracker.load_domains_from_file
+		if domain_tracker.domain_known?(root)
+			domain_tracker=nil
+			return true
+		else
+			domain_tracker=nil
+			return false
+		end
+	end
+
 	# Setter to add host entry to the cache once at a time
 	def add(host)
 		puts "Add entry to the local host repository: #{host}"
 		host=host.strip.downcase unless host.nil?
+		root=get_domain_root(host)
 		unless @known_hosts.key?(host)
 			ip=host_2_ip(host)
 			record=Hash.new
 			if is_ip?(ip)
 				# filter host to known domains only
-				root=get_domain_root(host)
-				puts "Domain root: #{root}" if @verbose
-				domain_tracker=Wmap::DomainTracker.instance
-				domain_tracker.data_dir=@data_dir
-				domain_tracker.domains_file = domain_tracker.data_dir + "domains"
-				domain_tracker.load_domains_from_file
-				if domain_tracker.domain_known?(root)
-					domain_tracker=nil
+				if is_trusted?(host)
 					record[host]=ip
 					record[ip]=host
 					puts "Host data repository entry loaded: #{host} <=> #{ip}"
 					# Replace instance with the class variable to avoid potential race condition under parallel engine
 					# add additional logic to update the sub-domain table as well, 02/10/2014
 					sub=get_sub_domain(host)
-					if sub!=root
+					if sub!=nil
 						tracker=Wmap::DomainTracker::SubDomain.instance
 						tracker.data_dir=@data_dir
 						tracker.sub_domains_file = tracker.data_dir + "sub_domains"
@@ -142,8 +155,8 @@ class Wmap::HostTracker
 		else
 			puts "Host is already exist. Skip: #{host}"
 		end
-	rescue => ee
-		puts "Exception on method #{__method__}: #{ee}" if @verbose
+	#rescue => ee
+	#	puts "Exception on method #{__method__}: #{ee}" if @verbose
 	end
 
 	# Setter to add host entry to the local hosts in batch (from an array)
@@ -164,7 +177,7 @@ class Wmap::HostTracker
 				end
 			end
 			@known_hosts.merge!(results)
-			puts "Done loading entries."
+			puts "Done loading host entries."
 			return results
 		else
 			puts "Error: empty list - no entry is loaded. Please check your input list and try again."

data/lib/wmap/site_tracker.rb

@@ -77,7 +77,9 @@ class Wmap::SiteTracker
 		f.write "# Local site store created by class #{self.class} method #{__method__} at: #{timestamp}\n"
 		f.write "# Website,Primary IP,Port,Hosting Status,Server,Response Code,MD5 Finger-print,Redirection,Timestamp\n"
 		@known_sites.keys.sort.map do |key|
-			f.write "#{key},#{@known_sites[key]['ip']},#{@known_sites[key]['port']},#{@known_sites[key]['status']},#{@known_sites[key]['server']},#{@known_sites[key]['code']},#{@known_sites[key]['md5']},#{@known_sites[key]['redirection']},#{@known_sites[key]['timestamp']}\n"
+			if is_trusted?(key)
+				f.write "#{key},#{@known_sites[key]['ip']},#{@known_sites[key]['port']},#{@known_sites[key]['status']},#{@known_sites[key]['server']},#{@known_sites[key]['code']},#{@known_sites[key]['md5']},#{@known_sites[key]['redirection']},#{@known_sites[key]['timestamp']}\n"
+			end
 		end
 		f.close
 		puts "site store table is successfully saved: #{file_sites}"
@@ -94,6 +96,24 @@ class Wmap::SiteTracker
 		puts "Exception on method #{__method__}: #{ee}"
 	end
 
+	# determine site is trusted based on the known domains
+	def is_trusted?(site)
+		trusted=false
+		host=url_2_host(site)
+		root=get_domain_root(host)
+		domain_tracker=Wmap::DomainTracker.instance
+		domain_tracker.data_dir=@data_dir
+		domain_tracker.domains_file=@data_dir + "/" + "domains"
+		File.write(domain_tracker.domains_file, "") unless File.exist?(domain_tracker.domains_file)
+		domain_tracker.load_domains_from_file(domain_tracker.domains_file)
+		trusted=domain_tracker.domain_known?(root)
+		domain_tracker=nil
+		return trusted
+	rescue => ee
+		puts "Exception on method #{__method__}: #{ee}"
+		return trusted
+	end
+
 	# Setter to add site entry to the cache one at a time
 	def add(site)
 		puts "Add entry to the site store: #{site}"
@@ -132,6 +152,10 @@ class Wmap::SiteTracker
 			end
 		end
 		# add record only if trusted
+		host_tracker = Wmap::HostTracker.instance
+		host_tracker.data_dir= @data_dir
+		host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
+		host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
 		if trusted
 			# Add logic to check site status before adding it
 			checker=Wmap::UrlChecker.new(:data_dir=>@data_dir).check(site)
@@ -144,10 +168,6 @@ class Wmap::SiteTracker
 				raise "Site is currently down. Skip #{site}" if checker['code']==10000
 			end
 			raise "Exception on add method - Fail to resolve the host-name: Host - #{host}, IP - #{ip}. Skip #{site}" unless is_ip?(ip)
-			host_tracker = Wmap::HostTracker.instance
-			host_tracker.data_dir= @data_dir
-			host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
-			host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
 			# Update the local host table when necessary
 			if is_ip?(host)
 				# Case #1: Trusted site contains IP
@@ -341,8 +361,8 @@ class Wmap::SiteTracker
 		else
 			puts "Error: no entry is loaded. Please check your list and try again."
 		end
-	#rescue => ee
-	#	puts "Exception on method #{__method__}: #{ee}" if @verbose
+	rescue => ee
+		puts "Exception on method #{__method__}: #{ee}" if @verbose
 	end
 	alias_method :dels, :bulk_delete
 

data/lib/wmap/site_tracker/wp_tracker.rb

@@ -174,7 +174,7 @@ class WpTracker < Wmap::SiteTracker
 				end
 			end
 			@known_wp_sites.merge!(results)
-			puts "Done loading entries."
+			puts "Done loading wp entries."
 			return results
 		else
 			puts "Error: no entry is loaded. Please check your list and try again."

data/lib/wmap/url_checker.rb

@@ -88,9 +88,9 @@ class Wmap::UrlChecker
 		checker['redirection']=nil
 		checker['timestamp']=timestamp
 		return checker
-	rescue Exception => ee
-		puts "Exception on method #{__method__} for #{url}: #{ee}" # if @verbose
-		return nil
+	#rescue Exception => ee
+	#	puts "Exception on method #{__method__} for #{url}: #{ee}" # if @verbose
+	#	return nil
 	end
 	alias_method :check, :url_worker
 

data/lib/wmap/url_crawler.rb

@@ -90,7 +90,7 @@ class Wmap::UrlCrawler
 
     # The worker instance of crawler who perform the labour work
 	def crawl_worker(url0)
-		puts "Please be aware that it may take a while to crawl #{url0}, depending on the site's responsiveness and the amount of contents."
+		puts "Please be aware that it may take a while to crawl #{url0}, depending on the site's responsiveness and discovery contents."
 		# Input URL sanity check first
 		if is_url?(url0)
 			host=url_2_host(url0)

data/lib/wmap/url_crawler/adware_tag.rb

@@ -119,7 +119,7 @@ module Wmap
 					end
 				end
 				@tag_store.merge!(results)
-				puts "Done loading entries."
+				puts "Done loading adware entries."
         tags = nil
 				return results
 			else

data/version.txt

@@ -3,8 +3,8 @@
 ###############################################################################
 package = wmap
 # wmap version 2.0 == web_discovery version 1.5.3
-version = 2.8.2
-date = 2021-07-23
+version = 2.8.3
+date = 2021-07-26
 
 author = Sam (Yang) Li
 email = yang.li@owasp.org

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wmap
 version: !ruby/object:Gem::Version
-  version: 2.8.2
+  version: 2.8.3
 platform: ruby
 authors:
 - Sam (Yang) Li
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-23 00:00:00.000000000 Z
+date: 2021-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dnsruby