wmap 2.4.4

data/lib/wmap/domain_tracker.rb

@@ -0,0 +1,342 @@
+#--
+# Wmap
+#
+# A pure Ruby library for the Internet web application discovery and tracking.
+#
+# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
+#++
+require "parallel"
+#require "singleton"
+
+
+# Class to track the known (trusted) Internet domains
+class Wmap::DomainTracker
+	include Wmap::Utils
+	#include Singleton
+
+
+	attr_accessor :verbose, :max_parallel, :domains_file, :file_domains, :data_dir
+	attr_reader :known_internet_domains
+
+	# Set default instance variables
+	def initialize (params = {})
+		# Initialize the instance variables
+		@verbose=params.fetch(:verbose, false)
+		@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
+		@file_domains=params.fetch(:domains_file, @data_dir+'domains')
+		@max_parallel=params.fetch(:max_parallel, 40)
+		# Hash table to hold the trusted domains
+		File.write(@file_domains, "") unless File.exist?(@file_domains)
+		@known_internet_domains=load_domains_from_file(@file_domains)
+		#@known_internet_sub_domains=Hash.new
+	end
+
+	# 'setter' to load the known Internet domains into an instance variable
+	def load_domains_from_file (file=@file_domains, lc=true)
+		puts "Loading trusted domain file: #{file}"	if @verbose
+		begin
+			known_internet_domains=Hash.new
+			f_domains=File.open(file, 'r')
+			f_domains.each_line do |line|
+				puts "Processing line: #{line}" if @verbose
+				line=line.chomp.strip
+				next if line.nil?
+				next if line.empty?
+				next if line =~ /^\s*#/
+				line=line.downcase if lc==true
+				entry=line.split(',')
+				if known_internet_domains.key?(entry[0])
+					next
+				else
+					if entry[1] =~ /yes/i
+						known_internet_domains[entry[0]]=true
+					else
+						known_internet_domains[entry[0]]=false
+					end
+				end
+
+			end
+			f_domains.close
+			return known_internet_domains
+		rescue => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+			return nil
+		end
+	end
+
+	# Save the current domain hash table into a file
+	def save_domains_to_file!(file_domains=@file_domains, domains=@known_internet_domains)
+		puts "Saving the current domains cache table from memory to file: #{file_domains} ..." if @verbose
+		begin
+			timestamp=Time.now
+			f=File.open(file_domains, 'w')
+			f.write "# Local domains file created by class #{self.class} method #{__method__} at: #{timestamp}\n"
+			f.write "# domain name, free zone transfer detected?\n"
+			domains.keys.sort.map do |key|
+				if domains[key]
+					f.write "#{key}, yes\n"
+				else
+					f.write "#{key}, no\n"
+				end
+			end
+			f.close
+			puts "Domain cache table is successfully saved: #{file_domains}"
+		rescue => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+		end
+	end
+	alias_method :save!, :save_domains_to_file!
+
+	# Count numbers of entries in the domain cache table
+	def count
+		puts "Counting number of entries in the domain cache table ..."
+		begin
+			cnt=0
+			@known_internet_domains.map do |key|
+				unless key =~ /\w+\.\w+/
+					cnt=cnt+1
+				end
+			end
+			puts "Current number of entries: #{cnt}"
+			return cnt
+		rescue => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+		end
+	end
+	alias_method :size, :count
+
+	# 'setter' to add domain entry to the cache one at a time
+	def add(host)
+		puts "Add entry to the local domains cache table: #{host}" if @verbose
+		#begin
+			host=host.strip.downcase
+			if @known_internet_domains.key?(host)
+				puts "Domain is already exist. Skipping: #{host}"
+			else
+				root=get_domain_root(host)
+				sub=get_subdomain(host)
+				record=Hash.new
+				if host == root
+					if zone_transferable?(root)
+						record[root]=true
+						#@known_internet_domains[root]=true
+					else
+						record[root]=false
+						#@known_internet_domains[root]=false
+					end
+					puts "Entry loaded: #{record}"
+					@known_internet_domains.merge!(record)
+					return record
+				elsif sub.nil?				# 2/10/2014, additional logic to support sub-domains
+					# do nothing
+				elsif host != sub
+					if zone_transferable?(sub)
+						#@known_internet_domains[sub]=true
+						record[sub]=true
+					else
+						#@known_internet_domains[sub]=false
+						record[sub]=false
+					end
+					puts "Entry loaded: #{record}"
+					@known_internet_domains.merge!(record)
+					return record
+				else
+					puts "Problem add domain #{host} - please use legal root domain or sub domain only."
+				end
+			end
+		#rescue => ee
+			#puts "Exception on method #{__method__}: #{ee}" if @verbose
+		#end
+	end
+
+	# 'setter' to add domain entry to the cache in batch (from a file)
+	def file_add(file)
+		begin
+			puts "Add entries to the local domains cache table from file: #{file}" if @verbose
+			raise "File non-exist. Please check your file path and name again: #{file}" unless File.exist?(file)
+			changes=Array.new
+			domains=file_2_list(file)
+			changes=bulk_add(domains)
+		rescue => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+		end
+	end
+
+	# 'setter' to add domain entry to the cache in batch (from a list)
+	def bulk_add(list, num=@max_parallel)
+		puts "Add entries to the local domains cache table from list: #{list}" if @verbose
+		begin
+			results=Hash.new
+			domains=list
+			if domains.size > 0
+				Parallel.map(list, :in_processes => num) { |target|
+					add(target)
+				}.each do |process|
+					if process.nil?
+						next
+					elsif process.empty?
+						#do nothing
+					else
+						results.merge!(process)
+					end
+				end
+				@known_internet_domains.merge!(results)
+				puts "Done loading entries."
+				return results
+			else
+				puts "Error: no entry is loaded. Please check your list and try again."
+			end
+			return results
+		rescue => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+		end
+	end
+	alias_method :adds, :bulk_add
+
+	# 'setter' to remove entry from the cache one at a time
+	def delete(domain)
+		puts "Remove entry from the domains cache table: #{domain} " if @verbose
+		begin
+			domain=domain.strip.downcase
+			if @known_internet_domains.key?(domain)
+				@known_internet_domains.delete(domain)
+				puts "Entry cleared: #{domain}"
+				return domain
+			else
+				puts "Entry not fund. Skipping: #{domain}"
+			end
+		rescue => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+		end
+	end
+
+	# 'setter' to delete domain entry to the cache in batch (from a list)
+	def bulk_delete(list)
+		puts "Delete entries to the local domains cache table from list: #{list}" if @verbose
+		begin
+			domains=list
+			changes=Array.new
+			if domains.size > 0
+				domains.map do |x|
+					domain=delete(x)
+					changes.push(domain) unless domain.nil?
+				end
+				puts "Done deleting domains from list: #{list}"
+				return changes
+			else
+				puts "Exception on method bulk_delete: no entry is loaded. Please check your list and try again."
+			end
+		rescue => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+		end
+	end
+	alias_method :dels, :bulk_delete
+
+	# 'setter' to delete domain entry to the cache in batch (from a file)
+	def file_delete(file)
+		begin
+			puts "Delete entries to the local domains cache table from file: #{file}" if @verbose
+			raise "File non-exist. Please check your file path and name again: #{file}" unless File.exist?(file)
+			domains=file_2_list(file)
+			changes=bulk_delete(domains)
+		rescue => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+		end
+	end
+
+	# 'setter' to remove all entries from the store
+	def delete_all
+		puts "Delete all entries in the domain store! " if @verbose
+		begin
+			@known_internet_domains.keys.map do |domain|
+				delete(domain)
+			end
+		rescue => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+		end
+	end
+
+	# Refresh the domain entry one at a time
+	def refresh(domain)
+		begin
+			abort "Trusted Internet domain file not loaded properly! " if @known_internet_domains.nil?
+			domain=domain.strip.downcase unless domain.nil?
+			if domain_known?(domain)
+				delete(domain)
+				add(domain)
+				return domain
+			else
+				puts "Unknown domain: #{domain}"
+				return nil
+			end
+		rescue => ee
+			puts "Exception on method #{__method__} for #{domain}: #{ee}" if @verbose
+			return nil
+		end
+	end
+
+	# Simple method to check if a domain is already within the domain cache table
+	def domain_known?(domain)
+		begin
+			#abort "Trusted Internet domain file not loaded properly! " if @known_internet_domains.nil? or @known_internet_sub_domains.nil?
+			domain=domain.strip.downcase unless domain.nil?
+			case self.class.name
+			when "Wmap::DomainTracker"
+				return @known_internet_domains.key?(domain)
+			when "Wmap::DomainTracker::SubDomain"
+				return @known_internet_sub_domains.key?(domain)
+			else
+				return nil
+			end
+		rescue => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+		end
+		return false
+	end
+	alias_method :is_known?, :domain_known?
+	alias_method :is_domain_known?, :domain_known?
+
+	# Dump out the list of known domains
+	def get_domains
+		puts "Retrieve a list of known domain ..." if @verbose
+		begin
+			return @known_internet_domains.keys
+		rescue Exception => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+			return nil
+		end
+	end
+	alias_method :dump_domains, :get_domains
+	alias_method :dump, :get_domains
+
+	# Search potential matching domains from the domain store by using simple regular expression. Note that any upper-case char in the search string will be automatically converted into lower case
+	def search (pattern)
+		puts "Search domain store for the regular expression: #{pattern}" if @verbose
+		begin
+			pattern=pattern.strip.downcase
+			results=Array.new
+			@known_internet_domains.keys.map do |key|
+				if key =~ /#{pattern}/i
+					results.push(key)
+				end
+			end
+			return results
+		rescue Exception => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+			return nil
+		end
+	end
+	alias_method :find, :search
+
+	# Print summary report on all known / trust domains in the domain cache table
+	def print_known_domains
+		puts "\nSummary of known Internet Domains:"
+		@known_internet_domains.keys.sort.each do |domain|
+			puts domain
+		end
+		puts "End of the summary"
+	end
+	alias_method :print, :print_known_domains
+
+	private :load_domains_from_file
+end

data/lib/wmap/geoip_tracker.rb

@@ -0,0 +1,72 @@
+#--
+# Wmap
+#
+# A pure Ruby library for Internet web application discovery and tracking.
+#
+# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
+#++
+require "geoip"
+
+
+# Wrapper class of the 'GeoIP' library - http://geoip.rubyforge.org/
+# For detail explanation of Geographic information of an IP address (GeoIP) and its data repository, please refer to the vendor MaxMind (http://www.maxmind.com)
+class Wmap::GeoIPTracker
+	include Wmap::Utils
+	
+	attr_accessor :db, :verbose
+	
+	# This product includes GeoLite data created by MaxMind, available from
+	# <a href="http://www.maxmind.com">http://www.maxmind.com</a>.
+	Db_city=File.dirname(__FILE__)+"/../../dicts/GeoLiteCity.dat"
+	Db_asn=File.dirname(__FILE__)+"/../../dicts/GeoIPASNum.dat"
+	Db_country=File.dirname(__FILE__)+"/../../dicts/GeoIP.dat"
+	
+	# Set default instance variables
+	def initialize (params = {})		
+		@verbose=params.fetch(:verbose, false)
+		@db=params.fetch(:db, Db_city)
+	end
+
+	# Wrapper for the Ruby GeoIP City class - return data structure below on successful lookup 
+	# Struct.new(:request, :ip, :country_code2, :country_code3, :country_name, :continent_code, :region_name, :city_name, :postal_code, :latitude, :longitude, :dma_code, :area_code, :timezone)
+	def city(object)	
+		puts "Perform GeoIP city lookup on: #{object}" if @verbose
+		begin
+			object=object.strip
+			raise "Unknown object format - only valid hostname or IP is accepted: #{object}" unless is_ip?(object) or is_fqdn?(object)
+			GeoIP.new(Db_city).city(object)
+		rescue Exception => ee
+			puts "Exception on method city: #{object}" if @verbose
+			return nil
+		end
+	end	
+	alias_method :query, :city
+
+	# Wrapper for the Ruby GeoIP Country class - return data structure below on successful lookup 
+	# Struct.new(:request, :ip, :country_code, :country_code2, :country_code3, :country_name, :continent_code)
+	def country(object)	
+		puts "Perform GeoIP country lookup on: #{object}" if @verbose
+		begin
+			object=object.strip
+			raise "Unknown object format - only valid hostname or IP is accepted: #{object}" unless is_ip?(object) or is_fqdn?(object)
+			GeoIP.new(Db_country).country(object)
+		rescue Exception => ee
+			puts "Exception on method country: #{object}" if @verbose
+			return nil
+		end
+	end	
+
+	# Wrapper for the Ruby GeoIP ASN class - return data structure below on successful lookup 
+	# Struct.new(:number, :asn)
+	def asn(object)	
+		puts "Perform GeoIP ASN lookup on: #{object}" if @verbose
+		begin
+			object=object.strip
+			raise "Unknown object format - only valid hostname or IP is accepted: #{object}" unless is_ip?(object) or is_fqdn?(object)
+			GeoIP.new(Db_asn).asn(object)
+		rescue Exception => ee
+			puts "Exception on method asn: #{object}" if @verbose
+			return nil
+		end
+	end	
+end

data/lib/wmap/google_search_scraper.rb

@@ -0,0 +1,177 @@
+#--
+# Wmap
+#
+# A pure Ruby library for Internet web application discovery and tracking.
+#
+# Copyright (c) 2012-2015 Yang Li 
+#++
+require 'open-uri'
+require 'nokogiri'
+
+
+# We build our own Google search class by querying Google search engine from its web interface, by simulating  
+# an anonymous web surfer. 
+# Note: we don't use the native Google API due to its pricing structure - We don't have budget for 
+#    this project, and we can not use the free version due to the limitation of 100 queries per day for free. See https://github.com/google/google-api-ruby-client for details.
+class Wmap::GoogleSearchScraper
+	include Wmap::Utils
+
+	attr_accessor :verbose, :http_timeout, :keyword_list
+	attr_reader :discovered_urls_from_scraper, :discovered_sites_from_scraper
+	
+	# Google search engine web interface locators
+	File_locator = File.dirname(__FILE__)+'/../../settings/google_locator.txt'
+	# Google search key words
+	File_keywords = File.dirname(__FILE__)+'/../../settings/google_keywords.txt'
+	
+
+	# Scraper default variables
+	def initialize (params = {})		
+		@verbose=params.fetch(:verbose, false)
+		@http_timeout=params.fetch(:http_timeout, 5000)
+		# Discovered data store		
+		@discovered_urls_from_scraper=Hash.new
+		@discovered_sites_from_scraper=Hash.new
+	end
+	
+	# Main worker method to simulate extensive google keyword searches on over 100+ countries and regions. The search will extract known web services related to the keyword by the Google Inc.
+	def google_worker (keyword)
+		begin
+			puts "Start the Google worker for: #{keyword}" if @verbose
+			links=Array.new
+			keyword=keyword.strip
+			google_locators = file_2_list(File_locator)
+			google_locators.map do |locator|
+				doc=google_search(locator,keyword) unless keyword.nil?
+				links+=extract_links(doc) unless doc.nil? 
+			end
+			return links.uniq.sort-["",nil]
+		rescue Exception => ee
+			puts "Exception on the method google_worker for #{keyword}: #{ee}" if @verbose
+			return nil
+		end	
+	end 
+	alias_method :worker, :google_worker
+	alias_method :search, :google_worker
+
+	# Main method to collect intelligences on the Google vast data warehouse. It works by hitting the Google engines with the keyword list. This exhausive method will sweep through the Google engines in over 100+ countries and regions one by one, in order to collect all related web service links collected by known the Google, Inc. across the global Internet.
+	def google_workers(keyword_list=file_2_list(File_keywords)) 
+		begin
+			puts "Start the Google worker for: #{keyword_list}" if @verbose
+			links=Array.new			
+			keyword_list.map do |keyword|
+				links+=google_worker(keyword)
+			end
+			return links.uniq.sort
+		rescue Exception => ee
+			puts "Exception on the method google_workers for #{keyword_list}: #{ee}" if @verbose
+			return nil
+		end	
+	end 
+	alias_method :workers, :google_workers
+	
+	# Perform a Google web interface keyword search, return as a Nokogiri::HTML:Document object for the search result page 
+	def google_search (locator,keyword)
+		begin
+			puts "Perform the keyword search on the Google web engine for: #{keyword}" if @verbose
+			link_search = locator + "search?q=" + URI::encode(keyword)
+			doc = Nokogiri::HTML(open(link_search))
+			return doc
+		rescue Exception => ee
+			puts "Exception on method google_search at Google engine location #{link_search} for the keyword #{keyword} : #{ee}" if @verbose
+		end
+	end
+	
+	# Search for nodes by css, and extract the hyper links
+	def extract_links (doc)
+		begin
+			puts "Extract the meaningful links from the DOC." if @verbose
+			links=Array.new
+			doc.css('a').each do |link|
+				ref=link.attribute('href').to_s
+				if ref =~ /\/url\?/
+					my_key=ref.sub(/\/url\?q\=/,'')
+					my_site=url_2_site(my_key)
+					links.push(my_key)
+					@discovered_urls_from_scraper[my_key]=true unless @discovered_urls_from_scraper.key?(my_key)
+					@discovered_sites_from_scraper[my_site]=true unless @discovered_sites_from_scraper.key?(my_site)
+				end
+			end
+			return links
+		rescue Exception => ee
+			puts "Exception on method extract_links: #{ee}" if @verbose
+			return nil
+		end 
+	end
+	
+	# Method to print out discovery URL result
+	def print_discovered_urls_from_scraper		
+		puts "Print discovered urls by the scraper. " if @verbose
+		begin
+			puts "\nSummary Report of Discovered URLs from the Scraper:"
+			@discovered_urls_from_scraper.keys.each do |url|
+				puts url
+			end
+			puts "Total: #{@discovered_urls_from_scraper.keys.size} url(s)"
+			puts "End of the summary"
+        rescue => ee
+			puts "Error on method print_discovered_urls_from_scraper: #{ee}" if @verbose
+        end
+	end	
+
+	# Method to print out discovery Sites result
+	def print_discovered_sites_from_scraper		
+		puts "Print discovered sites by the scraper. " if @verbose
+		begin
+			puts "\nSummary Report of Discovered Sites from the Scraper:"
+			@discovered_sites_from_scraper.keys.each do |site|
+				puts site
+			end
+			puts "Total: #{@discovered_sites_from_scraper.keys.size} site(s)"
+			puts "End of the summary"
+        rescue => ee
+			puts "Error on method print_discovered_sites_from_scraper: #{ee}" if @verbose
+        end
+	end	
+	
+	# 'getter' for the discovered sites from the Google search
+	def get_discovered_sites_from_scraper
+		puts "Getter for the discovered sites by the scraper. " if @verbose
+		begin
+			return @discovered_sites_from_scraper.keys.sort
+        rescue => ee
+			puts "Error on method get_discovered_sites_from_scraper: #{ee}" if @verbose
+        end
+	end
+	alias_method :print, :get_discovered_sites_from_scraper
+
+	# 'getter' for the discovered urls from the Google search
+	def get_discovered_urls_from_scraper
+		puts "Getter for the discovered urls by the scraper. " if @verbose
+		begin
+			return @discovered_urls_from_scraper.keys.sort
+        rescue => ee
+			puts "Error on method get_discovered_urls_from_scraper: #{ee}" if @verbose
+        end
+	end
+
+	# Save the discovered sites into a local file
+	def save_discovered_sites_from_scraper (file)
+		puts "Save the discovery result(sites) into a local file: #{file}" if @verbose
+		begin
+			f=File.open(file, 'w')
+			timestamp=Time.now
+			f.puts "# Discovery result written by Wmap::GoogleSearchScraper.save_discovered_sites_from_scraper method at #{timestamp}\n"
+			@discovered_sites_from_scraper.keys.sort.map { |x| f.puts "#{x}\n" }
+			f.close
+			raise "Unknown problem saving the result to file: #{file}" unless File.exist?(file)
+			puts "Done saving the discovery result into the local file: #{file}" 
+        rescue => ee
+			puts "Error on method save_discovered_sites_from_scraper: #{ee}" if @verbose
+        end
+	end
+	alias_method :save, :save_discovered_sites_from_scraper
+	
+	private
+
+end