wmap 2.4.4

data/lib/wmap/network_profiler.rb

@@ -0,0 +1,144 @@
+#--
+# Wmap
+#
+# A pure Ruby library for the Internet web application discovery and tracking.
+#
+# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
+#++
+require "net/ping"
+
+
+# Network profiler to optimize the port scanner performance for a specific network / IP. The ultimate goal is to set a reasonable socket time-out parameter for the scanners.
+class Wmap::NetworkProfiler
+	include Wmap::Utils
+	
+	attr_accessor :socket_timeout, :search_path, :max_parallel, :verbose
+	attr_reader :latency		# Discovered network latency 
+	File_discovery_ports=File.dirname(__FILE__)+'/../../settings/discovery_ports'	
+	
+	# Set default instance variables
+	def initialize (params = {})		
+		@verbose=params.fetch(:verbose, false)
+		@socket_timeout=params.fetch(:socket_timeout, 1500)
+		#@http_timeout=params.fetch(:http_timeout, 3000)
+		@search_path=["/sbin/","/usr/sbin/","/usr/local/bin/","/usr/bin/","/opt/bin/","/opt/sbin/"]
+		# Initialize the instance variables
+		@discovery_tcp_ports=params.fetch(:discovery_tcp_ports, file_2_list(File_discovery_ports).map!{|x| x.to_i} )
+	end
+
+	# Main worker method that determine the right profiling methods
+	def profile(host)	
+		puts "Perform web service discovery on host: #{host}" if @verbose
+		@latency = @socket_timeout
+		begin
+			if Process.euid == 0 && socket_icmp_pingable?(host)
+				puts "Network profiling by using raw socket ..." if @verbose
+			elsif shell_ping_exist? && shell_pingable?(host)
+				puts "Network profiling by using external shell ping program ..." if @verbose
+			elsif open_tcp_port?(host)
+				puts "Network profiling by using TCP ping ..." if @verbose
+			else
+				puts "No appropriate profiling method for #{host}" if @verbose
+				# Do nothing
+			end
+			puts "Found network latency for #{host}: #{@latency} ms" if @verbose
+			return @latency
+		rescue Exception => ee
+			puts "Exception on method #{__method__} for #{host}: #{ee}" if @verbose
+			return nil
+		end
+	end
+	
+	# Perform raw socket ICMP echo detection on the host. Note that socket ICMP packet manipulation 
+	#  need the root privilege access(for example, ICMP 'echo' need to snoop on the interface to detect any replies such as 'ECONNREFUSED'). 
+	#  That's why we also use external ping program for the normal users in case they do not has the access.
+	def socket_icmp_pingable? (target)
+		puts "Perform socket ICMP ping on the target: #{target}" if @verbose
+		begin
+			timeo = @socket_timeout/1000.0						# change time-out unit from sec to ms
+			p=Net::Ping::ICMP.new(target,nil,timeo) 
+			if p.ping
+				@latency=p.duration * 1000
+				puts "Socket ICMP echo test successful on #{target}." if @verbose
+				return true
+			else
+				puts "Socket ICMP echo test fail on #{target}." if @verbose
+				return false
+			end
+		rescue Exception => ee
+			puts "Error on method #{__method__} on target #{target}: #{ee}" if @verbose
+			return false
+		end
+	end
+	
+	# Wrapper for local ping executable. This is needed if the process do not have the root privilege to operate 
+	#   on the raw ICMP socket
+	def shell_pingable? (target)		
+		puts "Perform ping test from the shell on: #{target}" if @verbose
+		begin
+            sum=0
+			test_ping= `#{@which_ping} -c 3 #{target}`
+			test_ping.scan(/^(.+?)\stime=(.+)\s(.+?)\n/).map do |entry|
+				puts "entry: #{entry}" if @verbose
+				sum=sum+entry[1].to_f
+			end
+			if sum > 0
+				@latency = sum / 3
+				puts "Ping test from the shell environment successful on #{target}." if @verbose
+				return true
+			else
+				puts "Ping test from the shell environment fail on #{target}." if @verbose
+				return false
+			end
+		rescue Exception => ee
+			puts "Exception on method #{__method__} for #{host}: #{ee}" if @verbose
+			return false
+		end
+	end
+	
+	# Search for local ping executable program. This is helpful for the normal users who has no direct access to generate socket ICMP packets.
+	def shell_ping_exist? 
+		begin
+			puts "Search local shell environment for the ping program ..." if @verbose
+			@search_path.map do |path|
+				ping_exe=path+"ping"
+				if File.exist?(ping_exe) && File.executable?(ping_exe)
+					@which_ping=ping_exe
+					puts "Local ping program found: #{ping_exe}" if @verbose
+					return true
+				end
+			end
+			return false
+		rescue Exception => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+			return false
+		end
+	end
+
+	# Perform TCP Ping as a last resort of the network profiling effort, in case of ICMP tests fail.
+	def open_tcp_port? (target)
+		puts "Check if any TCP port in the list #{@discovery_tcp_ports} is open on the remote host: #{target}" if @verbose
+		begin
+			timeo = @socket_timeout/1000.0						# change time-out unit from sec to ms
+			p=Net::Ping::TCP.new(target,nil,timeo) 
+			@discovery_tcp_ports.map do |port|
+				p.port=port
+				if p.ping
+					@which_port=port	
+					# Bug in the current Net::Ping.ping module, where the 'duration' is 100 order off. We make it up here without fixing their code
+					@latency = p.duration * 1000 * 100
+					puts "TCP port detection successful on port: #{@which_port}" if @verbose
+					return true
+				end
+			end
+			puts "TCP port detection on remote host #{target} fail. " if @verbose
+			return false
+		rescue Exception => ee
+			puts "Error on method #{__method__} on target #{target}: #{ee}" if @verbose
+			return false
+		end
+	end
+	
+	private 
+	
+end

data/lib/wmap/port_scanner.rb

@@ -0,0 +1,208 @@
+#--
+# Wmap
+#
+# A pure Ruby library for the Internet web application discovery and tracking.
+#
+# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
+#++
+require "net/ping"
+require "parallel"
+
+# Port scanner class for the web application discovery and tracking
+class Wmap::PortScanner
+	include Wmap::Utils
+
+	attr_accessor :socket_timeout, :http_timeout, :discovery_tcp_ports, :max_parallel, :verbose
+	attr_reader :discovered_urls
+
+	# Use default common web service port list for the discovery
+	File_discovery_ports=File.dirname(__FILE__)+'/../../settings/discovery_ports'
+	# set hard limit of socket time-out to 3 seconds to avoid performance penalty
+	Max_socket_timeout=3000
+
+	# Set default instance variables
+	def initialize (params = {})
+		@verbose=params.fetch(:verbose, false)
+		@socket_timeout=params.fetch(:socket_timeout, 1500)
+		@http_timeout=params.fetch(:http_timeout, 5000)
+		@max_parallel=params.fetch(:max_parallel, 40)
+		# Initialize the instance variables
+		@discovery_tcp_ports=params.fetch(:discovery_tcp_ports, file_2_list(File_discovery_ports).map!{|x| x.to_i} )
+		@discovered_urls=Hash.new
+	end
+
+	# Pre-scan worker, to be used for network profiling to maximum the scan performance, for instance.
+	def pre_scan(host)
+		puts "Perform pre-scan works on host: #{host}" if @verbose
+		begin
+			# Use the following formula to 'guess' the right network time-out threshold for the scanner
+			nwk_to=Wmap::NetworkProfiler.new.profile(host)
+			if (100 + nwk_to*2).to_i > Max_socket_timeout
+				@socket_timeout=Max_socket_timeout
+			else
+				@socket_timeout=(100 + nwk_to*2).to_i
+			end
+			puts "Done with the pre-scan works: reset @socket_timeout to: #{@socket_timeout}" if @verbose
+		rescue Exception => ee
+			puts "Exception on method #{__method__} for #{host}: #{ee}" if @verbose
+			return nil
+		end
+	end
+
+	# Main worker method that run through the discovery ports list, check if any response to the HTTP request on the open ports, and finally return the findings in the URL format as an array
+	def scan (host)
+		puts "Perform web service discovery on host: #{host}"
+		begin
+			pre_scan(host)
+			urls=Array.new
+			@discovery_tcp_ports.map do |port|
+				if tcp_port_open?(host,port)
+					url=host_2_url(host,port)
+					urls.push(url) unless url.nil?
+				end
+			end
+			if urls.empty?
+				puts "No web service detected. "
+			else
+				urls.map do |url|
+					unless @discovered_urls.key?(url)
+						@discovered_urls[url]=true
+					end
+				end
+				puts "Detected web service on host #{host}: #{urls}"
+			end
+			return urls
+		rescue Exception => ee
+			puts "Exception on method #{__method__}  for #{host}: #{ee}" if @verbose
+			return nil
+		end
+	end
+	alias_method :query, :scan
+
+	# Parallel scanner - by utilizing fork manager 'parallel' to spawn numbers of child processes on multiple hosts/IPs simultaneously
+	def scans (targets,num=@max_parallel)
+		begin
+			urls=Array.new
+			# 10/5/2013 add additional logic to eliminate invalid /duplicate target(s)
+			targets -= ["", nil]
+			uniq_hosts=Hash.new
+			targets.dup.map do |target|
+				if is_fqdn?(target) or is_ip?(target)
+					ip=host_2_ip(target).to_s
+					if uniq_hosts.key?(ip)
+						targets.delete(target)
+					else
+						uniq_hosts[ip]=true
+					end
+				end
+			end
+			if targets.size > 0
+				puts "Start the parallel port scan on the target list:\n #{targets}"
+				Parallel.map(targets.shuffle, :in_processes => num) { |target|
+					scan(target)
+				}.each do |process|
+					if process.nil?
+						next
+					elsif process.empty?
+						#do nothing
+					else
+						process.map do |url|
+							unless @discovered_urls.key?(url)
+								@discovered_urls[url]=true
+							end
+						end
+						urls+=process
+					end
+				end
+			end
+			puts "Port scanning done successfully with the found web services: #{urls}"
+			return urls
+		rescue Exception => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+			return nil
+		end
+	end
+
+	# Parallel scans on a list of CIDRs from the input file, return the findings as the website construct within an array
+	def scan_file(file,num=@max_parallel)
+		puts "Start the parallel scans on the target file: #{file}"
+		begin
+			list=load_target_file(file)
+			urls=scans(list,num)
+		rescue Exception => ee
+			puts "Error on method #{__method__}: #{ee}" if @verbose
+			return nil
+		end
+		return urls
+	end
+	alias_method :file_scan, :scan_file
+
+	# Prepare and load the target list from a target file
+	def load_target_file (file)
+		puts "Preparing the discovery target file: #{file}" if @verbose
+		begin
+			targets=Array.new
+			f=File.open(file,'r')
+			f.each do |line|
+				line=line.chomp.strip
+				next if line.nil?
+				next if line.empty?
+				next if line =~ /^\s*#/
+				line=line.split(',')[0]
+				if is_cidr?(line)
+					ips=cidr_2_ips(line)
+					targets+=ips
+				elsif is_ip?(line) or is_fqdn?(line)
+					targets.push(line)
+				elsif is_url?(line)
+					host=url_2_host(line)
+					targets.push(host)
+				else
+					puts "Unknown entry in the seed file: #{line}"
+				end
+			end
+			f.close
+			return targets
+		rescue Exception => ee
+			puts "Error on method #{__method__} on file #{file} exception: #{ee}" if @verbose
+			return nil
+		end
+	end
+	alias_method :load, :load_target_file
+
+	# A simple TCP port scanner. This is the basic element of the port scanner. Notice the default time-out is set by the default instance variable @socket_timeout
+	def tcp_port_open? (host,port)
+		puts "Perform open port detection on: #{host}:#{port}, time-out: #{@socket_timeout} ms" if @verbose
+		#@socket_timeout = socket_timeout
+		timeo = @socket_timeout/1000.0						# change time-out unit from sec to ms
+		begin
+            if Net::Ping::TCP.new(host,port,timeo).ping
+				puts "Port open!" if @verbose
+                return true
+            else
+				puts "Port down." if @verbose
+                return false
+            end
+		rescue Exception => ee
+			puts "Exception on method #{__method__} for #{host}: #{ee}" if @verbose
+			return false
+		end
+	end
+
+	# Print out the summary report of discovered sites
+	def print_discovered_urls
+		puts "Print out port discovery results." if @verbose
+		puts "Summary of Discovered Sites:"
+		@discovered_urls.keys.sort.map { |x| puts x }
+		puts "End of Summary."
+	end
+	alias_method :print, :print_discovered_urls
+
+	# Count number of new found sites
+	def count
+		return @discovered_urls.size
+	end
+
+	private :load_target_file
+
+end

data/lib/wmap/site_tracker/deactivated_site.rb

@@ -0,0 +1,85 @@
+#--
+# Wmap
+#
+# A pure Ruby library for Internet web application discovery and tracking.
+#
+# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
+#++
+#require "singleton"
+
+
+# Class to trace de-activated site. This is need for basic state tracking for our sites.
+
+module Wmap
+class SiteTracker
+
+class DeactivatedSite < Wmap::SiteTracker
+	include Wmap::Utils
+	#include Singleton
+
+	attr_accessor :sites_file, :known_sites, :verbose, :data_dir
+
+	# Set default instance variables
+	def initialize (params = {})
+		# Initialize the instance variables
+		@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../../data/')
+		@f_sites=@data_dir + 'deactivated_sites'
+		@file_stores=params.fetch(:sites_file, @f_sites)
+		@verbose=params.fetch(:verbose, false)
+		# Hash table to hold the site store
+		File.write(@file_stores, "") unless File.exist?(@file_stores)
+		@known_sites=load_site_stores_from_file(@file_stores)
+	end
+
+	# Deactivate obsolete entrance from the live site store. Note this method is used by the parent class only
+	def add (site,entry)
+		begin
+			puts "Deactivate site: #{site}" if @verbose
+			@known_sites[site]=Hash.new unless @known_sites.key?(site)
+			@known_sites[site]['ip']=entry['ip']
+			@known_sites[site]['port']=entry['port']
+			@known_sites[site]['status']=entry['status']
+			@known_sites[site]['server']=entry['server']
+			@known_sites[site]['md5']=entry['md5']
+			@known_sites[site]['redirection']=entry['redirection']
+			@known_sites[site]['timestamp']=entry['timestamp']
+			@known_sites[site]['code']=entry['code']
+			puts "Deactivate site entry loaded: #{entry}"
+		rescue Exception => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+			return nil
+		end
+
+	end
+
+	# Refresh re-activated entrance in the store. Note this method is used by the parent class only
+	def delete (site)
+		begin
+			puts "Reactivate site: #{site}" if @verbose
+			site=site.strip.downcase unless site.nil?
+			@known_sites.delete(site)
+			puts "Site removed from the de-activated list."
+		rescue Exception => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+			return nil
+		end
+
+	end
+
+	# Procedures to discover deactivated sites from the live site store to here in one shot (TBD).
+	def update_from_site_store!
+		puts "Invoke internal procedures to update the site store."
+		begin
+		# To be further developed
+		rescue Exception => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+			return nil
+		end
+	end
+	alias_method :update!, :update_from_site_store!
+
+
+end
+
+end
+end