wmap 2.4.4 → 2.4.5

data/lib/wmap/cidr_tracker.rb

@@ -64,9 +64,11 @@ class Wmap::CidrTracker
 				@known_cidr_blks[key]['netname']=entry[2].nil? ? nil : entry[2].strip
 			end
 			f.close
-			# Sort the blocks in order once for better performance
-			@known_cidr_blks_desc_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>true)
-			@known_cidr_blks_asce_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>false)
+			# Sort the blocks in order once for better performance. Update 10/29/2018 to support Netaddr 2.x syntax
+			#@known_cidr_blks_desc_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>true)
+			#@known_cidr_blks_asce_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>false)
+			@known_cidr_blks_asce_index=@known_cidr_blks.keys.sort
+			@known_cidr_blks_desc_index=@known_cidr_blks_asce_index.reverse
 		rescue => ee
 			puts "Exception on method #{__method__}: #{ee}" # if @verbose
 		end
@@ -75,7 +77,7 @@ class Wmap::CidrTracker
 	# 'setter' to add an entry to CIDR store @known_cidr_blks
 	def add (cidr,ref=nil,netname=nil)
 		puts "Load the entry into the CIDR store: #{cidr}"
-		begin
+		#begin
 			raise "Unknown CIDR format: #{cidr}" unless is_cidr?(cidr)
 			# Obtain the 'ref' and 'netname' value automatically in case not passed as method parameters
 			if ref.nil? or netname.nil?
@@ -96,11 +98,13 @@ class Wmap::CidrTracker
 				puts "Entry loaded!"
 			end
 			# Re-sort the blocks in order for better performance
-			@known_cidr_blks_desc_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>true)
-			@known_cidr_blks_asce_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>false)
-		rescue => ee
-			puts "Exception on method #{__method__}: #{ee}" # if @verbose
-		end
+			#@known_cidr_blks_desc_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>true)
+			#@known_cidr_blks_asce_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>false)
+			@known_cidr_blks_asce_index=@known_cidr_blks.keys.sort
+			@known_cidr_blks_desc_index=@known_cidr_blks_asce_index.reverse
+		#rescue => ee
+		#	puts "Exception on method #{__method__}: #{ee}" # if @verbose
+		#end
 	end
 
 	# 'setter' to remove an entry to CIDR store @known_cidr_blks
@@ -117,8 +121,10 @@ class Wmap::CidrTracker
 				raise "Unknown CIDR entry: #{cidr}"
 			end
 			# Re-sort the blocks in order for better performance
-			@known_cidr_blks_desc_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>true)
-			@known_cidr_blks_asce_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>false)
+			#@known_cidr_blks_desc_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>true)
+			#@known_cidr_blks_asce_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>false)
+			@known_cidr_blks_asce_index=@known_cidr_blks.keys.sort
+			@known_cidr_blks_desc_index=@known_cidr_blks_asce_index.reverse
 		rescue => ee
 			puts "Exception on method #{__method__}: #{ee}" # if @verbose
 		end
@@ -167,6 +173,7 @@ class Wmap::CidrTracker
 			@known_cidr_blks_desc_index.each do |line|
 				first_octet_blk = line.split('.').first.to_i
 				next if first_octet_blk > first_octet_ip
+				puts "line: #{line}" if @verbose
 				cidr4 = NetAddr::CIDR.create(line)
 				known = cidr4.contains?(ip+'/32')
 				break if known
@@ -276,7 +283,7 @@ class Wmap::CidrTracker
 	# Save the current cidr hash table into a file
 	def save_cidrs_to_file!(file_cidrs=@file_cidr_seeds)
 		puts "Saving the current cidrs cache table from memory to file: #{file_cidrs} ..." if @verbose
-		begin
+		#begin
 			timestamp=Time.now
 			f=File.open(file_cidrs, 'w')
 			f.write "# Local cidrs file created by Wmap::CidrTracker.save method at: #{timestamp}\n"
@@ -288,9 +295,9 @@ class Wmap::CidrTracker
 			end
 			f.close
 			puts "CIDR cache table is successfully saved: #{file_cidrs}"
-		rescue => ee
-			puts "Exception on method #{__method__}: #{ee}" if @verbose
-		end
+		#rescue => ee
+		#	puts "Exception on method #{__method__}: #{ee}" if @verbose
+		#end
 	end
 	alias_method :save!, :save_cidrs_to_file!
 

data/lib/wmap/host_tracker/primary_host.rb

@@ -14,7 +14,7 @@ module Wmap
 	# Class to differentiate the primary host-name from the potential aliases. This is needed in order to minimize the confusion on our final site inventory list, as it contains a large number of duplicates (aliases). More specifically, a filter could be built by using this class to track the primary url of a website.
 	class PrimaryHost < Wmap::HostTracker
 		include Wmap::Utils
-		include Singleton
+		#include Singleton
 
 		attr_accessor :hosts_file, :verbose, :data_dir
 		attr_reader :known_hosts, :known_ips

data/lib/wmap/host_tracker.rb

@@ -42,7 +42,7 @@ class Wmap::HostTracker
 				entry=line.chomp.split(%r{\t+|\s+|\,})
 				key=entry[0].downcase
 				value=entry[1]
-				puts "Loading value pair: #{key} - #{value}" if @verbose
+				puts "Loading key value pair: #{key} - #{value}" if @verbose
 				known_hosts[key] = Hash.new unless known_hosts.key?(key)
 				known_hosts[key]= value
 				# For reverse host lookup
@@ -103,7 +103,7 @@ class Wmap::HostTracker
 	# Setter to add host entry to the cache once at a time
 	def add(host)
 		puts "Add entry to the local host repository: #{host}"
-		begin
+		#begin
 			host=host.strip.downcase unless host.nil?
 			unless @known_hosts.key?(host)
 				ip=host_2_ip(host)
@@ -137,9 +137,9 @@ class Wmap::HostTracker
 			else
 				puts "Host is already exist. Skip: #{host}"
 			end
-		rescue => ee
-			puts "Exception on method #{__method__}: #{ee}" if @verbose
-		end
+		#rescue => ee
+		#	puts "Exception on method #{__method__}: #{ee}" if @verbose
+		#end
 	end
 
 	# Setter to add host entry to the local hosts in batch (from an array)
@@ -521,7 +521,7 @@ class Wmap::HostTracker
 				entry=line.chomp.split(%r{\t+|\s+|\,})
 				key=entry[0].downcase
 				value=entry[1]
-				puts "Loading value pair: #{key} - #{value}" if @verbose
+				puts "Loading key value pair: #{key} - #{value}" if @verbose
 				host_store[key] = Hash.new unless known_hosts.key?(key)
 				host_store[key]= value
 			end

data/lib/wmap/site_tracker.rb

@@ -180,7 +180,7 @@ class Wmap::SiteTracker
 						if my_tracker.host_known?(host)
 							old_ip=my_tracker.local_host_2_ip(host)
 							if old_ip != ip
-								my_tracker.efresh(host)
+								my_tracker.refresh(host)
 								my_tracker.save!
 							else
 								puts "Host resolve to the same IP #{ip} - no need to update the local host table." if @verbose
@@ -205,7 +205,7 @@ class Wmap::SiteTracker
 					if my_tracker.host_known?(host)
 						old_ip=my_tracker.local_host_2_ip(host)
 						if old_ip != ip
-							my_tracker.efresh(host)
+							my_tracker.refresh(host)
 							my_tracker.save!
 						else
 							# Skip - no need to update the local hosts table
@@ -582,7 +582,7 @@ class Wmap::SiteTracker
 				host=url_2_host(key)
 				md5=@known_sites[key]['md5']
 				code=@known_sites[key]['code']
-				ip=my_trakcer.local_host_2_ip(host)
+				ip=my_tracker.local_host_2_ip(host)
 				ip=host_2_ip(host) if ip.nil?
 				# filtering out 'un-reachable' sites
 				next if (code == 10000 or code == 20000)
@@ -835,7 +835,7 @@ class Wmap::SiteTracker
 	# Retrieve the unique sites from the local site store in the primary host format
 	def get_prim_uniq_sites
 		puts "Retrieve and prime unique sites in the site store. " if @verbose
-		begin
+		#begin
 			host_tracker=Wmap::HostTracker.new(:data_dir=>@data_dir)
 			primary_host_tracker=Wmap::HostTracker::PrimaryHost.new(:data_dir=>@data_dir)
 			# Step 1. Retrieve the unique site list first
@@ -884,9 +884,9 @@ class Wmap::SiteTracker
 			primary_host_tracker=nil
 			host_tracker=nil
 			return prim_uniq_sites
-		rescue => ee
-			puts "Exception on method #{__method__}: #{ee}"
-		end
+		#rescue => ee
+		#	puts "Exception on method #{__method__}: #{ee}"
+		#end
 	end
 	alias_method :get_prime, :get_prim_uniq_sites
 

data/lib/wmap/url_crawler.rb

@@ -31,7 +31,7 @@ class Wmap::UrlCrawler
 	# Crawler instance default variables
 	def initialize (params = {})
 		@verbose=params.fetch(:verbose, false)
-		@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../logs/')
+		@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
 		@http_timeout=params.fetch(:http_timeout, 5000)
 		@crawl_depth=params.fetch(:crawl_depth, 4)
 		@crawl_page_limit=params.fetch(:crawl_page_limit, 1000)
@@ -224,11 +224,11 @@ class Wmap::UrlCrawler
 	alias_method :query_file, :crawl_workers_on_file
 	alias_method :crawl_file, :crawl_workers_on_file
 
-    # Wrapper for the OpenURI open method - create an open_uri object and return the reference upon success
+  # Wrapper for the OpenURI open method - create an open_uri object and return the reference upon success
 	def open_url(url)
-		puts "Open url #{url} by creating an open_uri object. Return the reference upon success." if @verbose
 		#url_object = nil
-        begin
+    begin
+			puts "Open url #{url} by creating an open_uri object. Return the reference upon success." if @verbose
 			if url =~ /http\:/i
 				# patch for allow the 'un-safe' URL redirection i.e. https://www.example.com -> http://www.example.com
 				url_object = open(url, :allow_redirections=>:safe, :read_timeout=>Max_http_timeout/1000)
@@ -240,24 +240,24 @@ class Wmap::UrlCrawler
 				raise "Invalid URL format - please specify the protocol prefix http(s) in the URL: #{url}"
 			end
 			return url_object
-        rescue => ee
-            puts "Exception on method #{__method__} for #{url}: #{ee}" if @verbose
-            return nil
-        end
+    rescue => ee
+      puts "Exception on method #{__method__} for #{url}: #{ee}" if @verbose
+      return nil
     end
+  end
 
 	# Wrapper to use OpenURI method 'read' to return url body contents
 	def read_url(url)
-		puts "Wrapper to return the OpenURI object for url: #{url}" if @verbose
 		begin
+			puts "Wrapper to return the OpenURI object for url: #{url}" if @verbose
 			url_object=open_url(url)
 			@visited_urls_by_crawler[url]=true unless @visited_urls_by_crawler.key?(url)
 			body=url_object.read
 			return body
-        rescue => ee
-            puts "Exception on method #{__method__}: #{ee}" if @verbose
-            return nil
-        end
+  	rescue => ee
+      puts "Exception on method #{__method__}: #{ee}" if @verbose
+      return nil
+    end
 	end
 
     # Return the destination url in case of url re-direct
@@ -268,11 +268,11 @@ class Wmap::UrlCrawler
 				return url_object.base_uri.to_s
 			end
 			return url
-        rescue => ee
-            puts "Exception on method #{__method__}: #{ee}" if @verbose
-            return nil
-        end
+    rescue => ee
+      puts "Exception on method #{__method__}: #{ee}" if @verbose
+      return nil
     end
+  end
 
     # Wrapper for the Nokogiri DOM parser
 	def parse_html(html_body)

data/lib/wmap/utils/domain_root.rb

@@ -8,25 +8,29 @@
 
 
 module Wmap
- module Utils	
+ module Utils
   # Module to validate and retrieve the top or second level domain name from a host-name (FQDN).
-  module DomainRoot	
+  module DomainRoot
 	extend self
 	# Internet Domain Architecture Definitions
 	File_ccsld=File.dirname(__FILE__)+'/../../../dicts/ccsld.txt'
 	File_cctld=File.dirname(__FILE__)+'/../../../dicts/cctld.txt'
 	File_gtld=File.dirname(__FILE__)+'/../../../dicts/gtld.txt'
-	
+  File_tld=File.dirname(__FILE__)+'/../../../dicts/tlds.txt'
+
 	# Main function to retrieve the registered domain ('domain root' from the 'registrant' perspective) from a hostname, for example, "www.telegraph.co.uk" -> "telegraph.co.uk"
 	def get_domain_root (host)
 		puts "Retrieve the root domain for host: #{host}" if @verbose
 		begin
+      # Comnplete Top Level Domain List - loading once
+      @tlds=file_2_hash(File_tld) if @tlds.nil?
 			# Generic Top Level Domain List - loading once
 			@gtld=file_2_hash(File_gtld) if @gtld.nil?
 			# Country code top-level domain list - loading once
 			@cctld=file_2_hash(File_cctld) if @cctld.nil?
 			# Country code second level domain - loading once
 			@ccsld=load_ccsld_from_file(File_ccsld) if @ccsld.nil?
+
 			if host.strip.nil?
 				puts "Error: empty record found. Please check your input and remove any empty line." if @verbose
 				return nil
@@ -35,15 +39,15 @@ module Wmap
 			end
 			found_tld=false
 			found_cctld=false
-			# search the general top level domain list first
+			# search the  top level domain list first
 			root_domain=""
 			dn=host.split(".")
-			if @gtld.key?(dn.last) 
-				found=false
-				if @cctld.key?(dn[dn.length-2]) 
-					found=true
+			if @tlds.key?(dn.last)
+				cc_found=false
+				if @cctld.key?(dn[dn.length-2])
+					cc_found=true
 				end
-				if found
+				if cc_found
 					root_domain=dn[dn.length-3] + "." + dn[dn.length-2] + "." + dn.last
 				else
 					root_domain=dn[dn.length-2] + "." + dn.last
@@ -51,7 +55,7 @@ module Wmap
 				found_tld=true
 			end
 			# search the country code top level domain list secondly
-			if @cctld.key?(dn.last) 
+			if @cctld.key?(dn.last)
 				found=false
 				# reverse search of general top level domain
 				if @gtld.key?(dn[dn.length-2])
@@ -65,8 +69,8 @@ module Wmap
 							break
 						end
 					end
-					# 1/8/2015: additional logic to handle invalid ccsld string: reserved gtld string 
-					#unless found 
+					# 1/8/2015: additional logic to handle invalid ccsld string: reserved gtld string
+					#unless found
 					#	if @gtld.key?(dn[dn.length-2])
 					#		puts "Invalid ccsld: #{dn[dn.length-2]} for host: #{host}"
 					#		return nil
@@ -77,17 +81,17 @@ module Wmap
 					root_domain=dn[dn.length-3] + "." + dn[dn.length-2] + "." + dn.last
 				else
 					root_domain=dn[dn.length-2] + "." + dn.last
-				end	
+				end
 				found_cctld=true
 			end
-			unless (found_tld or found_cctld)  
+			unless (found_tld or found_cctld)
 				puts "#{host} - the top level domain is unknown. Please check out your record #{root_domain} " if @verbose
 				return nil
 			else
 				puts "Domain root found: #{root_domain}" if @verbose
 				return root_domain
 			end
-		rescue => ee			
+		rescue => ee
 			puts "Exception on method #{__method__}: #{ee}" if @verbose
 			return nil
 		end
@@ -96,7 +100,7 @@ module Wmap
 	alias_method :root_domain, :get_domain_root
 	alias_method :domain_root, :get_domain_root
 	alias_method :host_2_domain, :get_domain_root
-	
+
 	# 'setter' to parse and load the known country code second level domain table from the file
 	# data structure example: {"uk" =>["co","plc"],"za"=>["mil","nom","org"]}
 	def load_ccsld_from_file (file_ccsld)
@@ -107,10 +111,10 @@ module Wmap
 			f.each do |line|
 				next unless line =~ /^\s+\.\w/
 				line=line.chomp.strip.downcase
-				entry=line.split(' ')[0].split('.')				
+				entry=line.split(' ')[0].split('.')
 				if entry.length > 2
 					key=entry.last
-					ccsld[key] = Array.new if not ccsld.key?(key)				
+					ccsld[key] = Array.new if not ccsld.key?(key)
 					val=entry[entry.length-2]
 					#puts "Loading country code second level domain table with - Country code: #{key}, Second level domain: #{val}" if @verbose
 					ccsld[key].push(val) unless key.nil?
@@ -119,7 +123,7 @@ module Wmap
 			f.close
 			# Sort the blocks once in descendant order once for better performance
 			return ccsld
-		rescue => ee			
+		rescue => ee
 			puts "Exception on method #{__method__}: #{ee}" if @verbose
 		end
 	end
@@ -158,15 +162,15 @@ module Wmap
 		rescue Exception => ee
 			puts "Exception on method #{__method__} for #{host}: #{ee}" if @verbose
 			return nil
-		end		
+		end
 	end
 	alias_method :get_subdomain, :get_sub_domain
-	
+
 	# Function to print instance variable - General top level domain list
 	def print_gtld
 		puts @gtld
 	end
-	
+
 	# Function to print instance variable - Country code top-level domain list
 	def print_cctld
 		puts @cctld
@@ -176,9 +180,9 @@ module Wmap
 	def print_ccsld
 		puts @ccsld
 	end
-	
+
 	private :load_ccsld_from_file
-	 
+
   end
  end
 end