wmap 2.4.4

data/lib/wmap/utils/utils.rb

@@ -0,0 +1,333 @@
+#--
+# Wmap
+#
+# A pure Ruby library for the Internet web application discovery and tracking.
+#
+# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
+#++
+
+require "resolv"
+require "netaddr"
+
+# Main utility module to provide the common functions across different classes
+module Wmap
+  module Utils	
+	include Wmap::Utils::DomainRoot
+	include Wmap::Utils::UrlMagic
+	include Wmap::Utils::Logger
+	extend self
+	
+	# Load entries from a text file and return an array
+	def file_2_list(f,lc=true)
+		puts "Loading records from file: #{f}" if @verbose
+		begin
+			list=Array.new
+			file = File.open(f, "r")
+			file.each_line do |line|
+				line=line.chomp.strip
+				next if line.nil?
+				next if line.empty?
+				next if line =~ /^\s*#/ 
+				line=line.downcase if lc==true
+				list.push(line.chomp.strip)
+			end
+			file.close
+			return list
+		rescue => ee
+			puts "Exception on method #{__method__} for file #{f}: #{ee}" if @verbose
+			return nil
+		end
+	end
+
+	# Save an array into a file
+	def list_2_file (list,file)
+		puts "Save list #{list} to plain file #{file}" if @verbose
+		begin
+			f = File.open(file, "w")
+			list.map do |ent|
+				#ent.strip!
+				# Append the unix line break
+				f.write("#{ent}\n")
+			end
+			f.close
+		rescue => ee
+			puts "Exception on method #{__method__} for file #{file}: #{ee}" if @verbose
+			return nil
+		end
+	end
+	
+	# Load entries from a text file and return a hash
+	def file_2_hash(f,lc=true)
+		puts "Loading records from file: #{f}" if @verbose
+		begin
+			hs=Hash.new
+			file = File.open(f, "r")
+			file.each_line do |line|
+				line=line.chomp.strip
+				next if line.nil?
+				next if line.empty?
+				line=line.downcase if lc==true
+				next if line =~ /^\s*#/ 
+				hs[line]=true unless hs.key?(line)
+			end
+			file.close
+			return hs
+		rescue => ee
+			puts "Exception on method #{__method__} on #{f}: #{ee}" if @verbose
+			return nil
+		end
+	end
+	
+	# Query the name-server to see if the dns record is still valid
+	def valid_dns_record? (hostname)
+		puts "Validate the hostname record: #{hostname}" if @verbose
+		begin
+			ips=Resolv.getaddresses(hostname)
+			if ips.empty?
+				return false
+			else
+				puts "Found: #{hostname}" if @verbose
+				return true
+			end
+		rescue => ee
+			puts "Exception on method #{__method__} for host #{hostname}: #{ee}" if @verbose
+			return false
+		end
+	end
+	alias_method :is_record? , :valid_dns_record?
+	alias_method :a_record? , :valid_dns_record?
+
+	# Test the DNS server if zone transfer is allowed. If allowed, save the found hosts into the class variable.
+	def zone_transferable?(domain)
+		puts "Check if the domain allows free zone transfer: #{domain}" 
+		begin
+			transferable=false
+			domain=domain.downcase
+			nameservers = get_nameservers(domain)
+			raise "Unable to determine the name servers for domain #{domain}" if nameservers.nil?
+			puts "Retrieved name servers: #{nameservers}" if @verbose
+			nameservers.each do |nssrv|
+				begin
+					puts "Attempt zone transfer on name server: #{nssrv}" 
+					if nssrv.nil?
+						abort "Method input variable error: no name server found!" if @verbose
+						next
+					end		
+					zt = Dnsruby::ZoneTransfer.new
+					zt.server=(nssrv) if nssrv!=""			
+					records = zt.transfer(domain)
+					if records==nil
+						puts "#{domain} zone transfer is not allowed on name server: #{nssrv}" 
+						next
+					else
+						puts "#{domain} zone transfer is allowed!"
+						transferable=true
+					end
+				rescue Exception=>ee
+					puts "Exception on method zone_transferable? for domain #{domain}: #{ee}" 
+				end
+			end
+			return transferable
+		rescue Exception => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+			return nil
+		end	
+	end	
+	
+	# Test if it's a legitimate IP4 address
+	def is_ip? (ip)
+		puts "Validate the IP format is valid: #{ip}" if @verbose
+		begin
+			ip=ip.strip
+			raise "This is an URL: #{ip}" if is_url?(ip)
+			if ip =~ /\d+\.\d+\.\d+.\d+/ and ip !~ /\/\d+/
+				octs=ip.split('.')
+				return false unless octs.size==4
+				octs.map { |x| return false unless x.to_i >=0 and x.to_i <=255 }
+			else
+				return false
+			end
+			puts "Confirmed as a valid IP: #{ip}" if @verbose
+			return true
+		rescue => ee
+			puts "Exception on method is_ip? for #{ip}: #{ee}" if @verbose
+			return false
+		end
+	end
+	alias_method :is_valid_ip?, :is_ip?
+	
+	# Simple test a host string format. Return true if it contains a valid internet domain sub-string. Note: Don't be confused with another method 'valid_dns_record?', which is a stricter and time-consuming test on the DNS server for a resolvable internet host. 
+	def is_fqdn? (host)
+		puts "Validate the host-name format is valid: #{host}" if @verbose
+		begin
+			return false if is_ip?(host) or is_url?(host)
+			domain=get_domain_root(host)
+			if domain.nil?
+				return false
+			elsif is_domain_root?(domain)
+				return true
+			else
+				return false
+			end
+		rescue => ee
+			puts "Exception on method is_fqdn? for #{host}: #{ee}" if @verbose
+			return false
+		end
+	end
+	alias_method :is_host?, :is_fqdn?
+
+	# Simple test to determine if the entry is in valid network cidr format
+	def is_cidr?(cidr)
+		puts "Validate if the entry is valid CIDR format: #{cidr}" if @verbose
+		begin
+		cidr=cidr.strip
+			if cidr =~  /^(\d+\.\d+\.\d+.\d+)\/(\d+)$/
+				ip=$1
+				mask=$2.to_i
+				if is_ip?(ip)
+					if mask >0 and mask <=32
+						puts "confirmed as a valid CIDR entry: #{cidr}" if @verbose
+						return true
+					else
+						return false
+					end
+				else
+					return false
+				end
+			else
+				return false
+			end
+		rescue => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+			return false
+		end
+	end
+	
+	# Sort an array of IPs in the ascendant order
+	def sort_ips (ips)
+		begin
+			"Sort the list of IP address in the ascendant order: #{ips}" if @verbose
+			return NetAddr.sort(ips)
+		rescue => ee			
+			puts "Exception on method sort_ips for IPs #{ips}: #{ee}" # if @verbose
+		end		
+	end
+	
+	# Perform the DNS query on a hostname over the Internet. Return the resolved IP(s) in an array
+	def host_2_ips (hostname)
+		begin
+			ips=Array.new
+			if is_ip?(hostname)
+				ips.push(hostname)
+				return ips
+			else
+				ips = Resolv.getaddresses(hostname)
+				if (ips.empty?) then
+					puts "Failed to resolve #{hostname}" if @verbose
+					return nil
+				else
+					return ips
+				end
+			end
+		rescue => ee
+			puts "Exception on method host_2_ips for host #{hostname}: #{ee}" if @verbose
+			return nil
+		end
+	end	
+	alias_method :ns_lookup, :host_2_ips
+
+	# Perform DNS query on a hostname.  Return the first resolved IP as a string
+	def host_2_ip (hostname)
+		puts "Perform DNS query on host: #{hostname}" if @verbose
+		begin
+			ips=Array.new
+			if is_ip?(hostname)
+				puts "No change - same IP is returned. " if @verbose
+				return hostname.strip
+			else
+				ips=Resolv.getaddresses(hostname)
+				if (ips.empty?) then
+					puts "Failed to resolve #{hostname}" if @verbose
+					return nil
+				else
+					puts "IP found: #{ips.first}" if @verbose
+					return ips.first.strip
+				end
+			end
+		rescue => ee
+			puts "Exception on method host_2_ip for host #{hostname}: #{ee}" if @verbose
+			return nil
+		end
+	end	
+	
+	# Retrieve a list of the authoritative name servers from the Internet whois data repository for the host / subdomain / domain
+	def get_nameservers (host)
+		puts "Retrieve a list of authoritative name server for: #{host}" if @verbose
+		begin
+			domain=get_domain_root(host)
+			w=Wmap::Whois.new
+			ns = w.query(domain).nameservers.map! { |x| x.name }
+			if ns.empty?
+				puts "No name server found for domain root: #{domain}" if @verbose
+				return nil			
+			else
+				return ns
+			end
+		rescue => ee
+			puts "Exception on method get_nameservers for #{host}: #{ee}" if @verbose
+			return nil
+		end
+	end
+
+	# Retrieve the first name server from the Internet whois data repository for the host / subdomain / domain
+	def get_nameserver (host)
+		puts "Retrieve the first authoritative name server for: #{host}" if @verbose
+		begin
+			domain=get_domain_root(host)
+			w=Wmap::Whois.new
+			ns = w.query(domain).nameservers.map! { |x| x.name }
+			if ns.empty?
+				puts "No name server found for domain root: #{domain}" if @verbose
+				return nil			
+			else
+				return ns.first
+			end
+		rescue => ee
+			puts "Exception on method get_nameservers for #{host}: #{ee}" if @verbose
+			return nil
+		end
+	end
+	alias_method :get_ns, :get_nameserver
+	
+	# Perform reverse dns lookup for an IP. Return the found 'hostname' if found, or the original IP if not
+	def reverse_dns_lookup (ip)
+		puts "Retrieve the hostname by the reverse DNS lookup on IP: #{ip}"
+		hostname = ip
+		begin 
+			hostname = Socket.gethostbyaddr(ip.split('.').collect{ |x| x.to_i}.pack("CCCC"))[0]
+			return hostname.downcase
+		rescue => ee
+			puts "Exception on method reverse_dns_lookup: #{ee}" if @verbose
+			return hostname
+		end		
+	end
+	alias_method :ip_2_host, :reverse_dns_lookup
+	
+	# Convert a CIDR to a list of IPs:  Input is a CIDR expression such as '192.168.1.1/30', output is an array of IPs
+	def cidr_2_ips (cidr)
+		puts "Method to convert a CIDR block into a list of IPs: #{cidr}" if @verbose
+		begin
+			cidr4 = NetAddr::CIDR.create(cidr)
+			ips = cidr4.enumerate(:Limit => 0, :Bitstep => 1)
+			#ips2 = ips.slice!(1, (ips.length-2))
+			return ips
+		rescue => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+			return nil
+		end	
+	end
+
+
+	 
+  end
+end

data/lib/wmap/whois.rb

@@ -0,0 +1,76 @@
+#--
+# Wmap
+#
+# A pure Ruby library for Internet web application discovery and tracking.
+#
+# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
+#++
+require "whois"
+
+# Wrapper class of the 'ruby-whois' library
+class Wmap::Whois
+	include Wmap::Utils
+	
+	attr_accessor :timeout, :verbose
+	
+	# Set default instance variables
+	def initialize (params = {})		
+		@verbose=params.fetch(:verbose, false)
+		@timeout=params.fetch(:timeout, 10)
+	end
+
+	# Wrapper for the Ruby Whois client class
+	def lookup(object)	
+		puts "Perform whois lookup on: #{object}" if @verbose
+		return Whois.lookup(object)
+	end	
+	alias_method :query, :lookup
+	
+	# Method to extract the netname information from the whois data repository query for an IP
+	def get_netname (ip)
+		puts "Perform whois lookup on an IP address. Then extract the netname from the query result for the IP: #{ip}" if @verbose
+		begin 
+			ip.strip!
+			raise "Unknown IP/CIDR format: #{ip}" unless is_ip?(ip) or is_cidr?(ip)
+			content_to_parse=query(ip).to_s
+			if content_to_parse =~ /^netname:(.+)\n/i
+				return $1.strip
+			elsif content_to_parse =~ /^.+\((NET\-.+)\).+\n/i
+				return $1.strip
+			else
+				return "UNKNOWN"
+			end
+			return "UNKNOWN"
+		rescue Exception => ee
+			puts "Exception on method get_netname: #{ee}" if @verbose
+			return "UNKNOWN"		
+		end
+	end
+
+	# Method to extract the netname description from the whois data repository query for an IP
+	def get_net_desc (ip)
+		puts "Perform whois lookup on an IP address. Then extract the netname description from the query result for the IP: #{ip}" if @verbose
+		begin 
+			ip.strip!
+			raise "Unknown IP/CIDR format: #{ip}" unless is_ip?(ip) or is_cidr?(ip)
+			desc=String.new
+			content_to_parse=query(ip).to_s
+			content_to_parse.scan(/^descr:(.+)\n/i).flatten.map do |entry|
+				desc=desc + " " + entry.strip
+			end
+			if desc.empty?
+				if content_to_parse =~ /^(.+)\((NET\-.+)\).+\n/i
+					desc=$1.strip
+				elsif content_to_parse =~ /^OrgName:(.+)\n/i
+					desc=$1.strip
+				else
+					desc="UNKNOWN"
+				end
+			end
+			return desc
+		rescue Exception => ee
+			puts "Exception on method get_net_desc: #{ee}" if @verbose
+			return "UNKNOWN"		
+		end
+	end
+end

data/lib/wmap.rb

@@ -0,0 +1,227 @@
+#--
+# Wmap
+#
+# A pure Ruby library for the Internet web application discovery and tracking.
+#
+# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
+#++
+require 'wmap/utils/domain_root'
+require 'wmap/utils/url_magic'
+require 'wmap/utils/logger'
+require 'wmap/utils/utils'
+require 'wmap/cidr_tracker'
+require 'wmap/domain_tracker'
+require 'wmap/domain_tracker/sub_domain'
+require 'wmap/host_tracker'
+require 'wmap/host_tracker/primary_host'
+require 'wmap/whois'
+require 'wmap/url_checker'
+require 'wmap/network_profiler'
+require 'wmap/port_scanner'
+require 'wmap/url_crawler'
+require 'wmap/dns_bruter'
+require 'wmap/site_tracker'
+require 'wmap/site_tracker/deactivated_site'
+require 'wmap/geoip_tracker'
+require 'wmap/google_search_scraper'
+
+module Wmap
+
+  NAME            = "Wmap"
+  GEM             = "wmap"
+  VERSION		  = File.dirname(__FILE__) + "/../version.txt"
+
+  class << self
+	attr_accessor :known_internet_domains
+	attr_writer   :verbose
+
+	# Simple parser for the project version file
+	def read_ver
+		ver=Hash.new
+		f=File.open(VERSION,'r')
+		f.each do |line|
+			line.chomp!
+			case line
+			when /^(\s)*#/
+				next
+			when /\=/
+				entry=line.split("=").map! {|x| x.strip}
+				ver[entry[0]]=entry[1]
+			end
+		end
+		f.close
+		return ver
+	end
+
+	# Project banner in ASCII Art 'soft' format, courtesy to http://patorjk.com/software/taag/
+	def banner
+		ver=read_ver
+		art=",--.   ,--.       ,--.       ,--.   ,--.
+|  |   |  | ,---. |  |-.     |   `.'   | ,--,--. ,---.  ,---.  ,---. ,--.--.
+|  |.'.|  || .-. :| .-. '    |  |'.'|  |' ,-.  || .-. || .-. || .-. :|  .--'
+|   ,'.   |\   --.| `-' |    |  |   |  |\ '-'  || '-' '| '-' '\   --.|  |
+'--'   '--' `----' `---'     `--'   `--' `--`--'|  |-' |  |-'  `----'`--'
+                                                `--'   `--'                  "
+		string = "-"*80 + "\n" + art + "\n" + "Version: " + ver["version"] + "\tRelease Date: " + ver["date"] + "\nDesigned and developed by: " + ver["author"] + "\nEmail: " + ver["email"] + "\tLinkedIn: " + ver["linkedin"] + "\n" + "-"*80
+	end
+
+	# Explorer to discover and inventory web application / service automatically
+	def wmap(seed)
+		cmd="bin/wmap" + " " + seed
+		system(cmd)
+	end
+
+	# Crawler to search url contents for new sites
+	def crawl(url)
+		crawler=Wmap::UrlCrawler.new
+		crawler.crawl(url)
+	end
+
+	# whois query and sort the result into structured data
+	def whois(domain)
+		whois=Wmap::Whois.new(:verbose=>false)
+		whois.query(domain)
+	end
+
+	# Fast tcp port scanner on a single host or IP
+	def scan(host)
+		scanner=Wmap::PortScanner.new
+		scanner.scan(host)
+	end
+
+	# Fast multi-processes tcp port scanner on a list of targets
+	def scans(target_list)
+		scanner=Wmap::PortScanner.new
+		scanner.scans(target_list)
+	end
+
+	# CIDR Tracking - check the host against the local CIDR seed file, return the CIDR tracking path if found
+	def track(host)
+		tracker=Wmap::CidrTracker.new
+		tracker.cidr_worker(host)
+	end
+
+	# GeoIP Tracking - check the host / IP against the GeoIP data repository, return the Geographic information if found
+	def geoip(host)
+		tracker=Wmap::GeoIPTracker.new
+		tracker.query(host)
+	end
+
+	# URL checker - check the status of the remote URL
+	def check(url)
+		checker=Wmap::UrlChecker.new(:verbose=>false)
+		checker.url_worker(url)
+	end
+
+	# Check if the IP is within the range of the known CIDR blocks
+	def ip_trusted?(ip)
+		tracker=Wmap::CidrTracker.new
+		tracker.ip_trusted?(ip)
+	end
+
+	# Domain Tracking - check with the trust domain seed file locally, to determine if it's a new internet domain
+	# NOT to confuse with the Internet 'whois' lookup
+	def domain_known?(domain)
+		tracker=Wmap::DomainTracker.new
+		tracker.domain_known?(domain)
+	end
+
+	# Host Tracking - check local hosts file to see if this is a hostname known from the host seed file
+	# NOT to confuse with a regular DNS lookup over the internet
+	def host_known?(host)
+		tracker=Wmap::HostTracker.new.host_known?(host)
+	end
+
+	# Sub-domain tracking - check local hosts file to see if the sub-domain is already known
+	def sub_domain_known?(host)
+		tracker=Wmap::HostTracker.new.sub_domain_known?(host)
+	end
+
+	# IP Tracking - check local hosts file to see if this is an IP known from the seed file
+	# NOT to confuse with a regular reverse DNS lookup over the internet
+	def ip_known?(ip)
+		tracker=Wmap::HostTracker.new.ip_known?(ip)
+	end
+
+	# DNS Brute Forcer
+	def dns_brute(domain)
+		bruter=Wmap::DnsBruter.new
+		bruter.query(domain)
+	end
+
+	# Retrieve root domain from a host
+	def domain_root(host)
+		Wmap::Utils.get_domain_root(host)
+	end
+
+	# Log the information into file
+	def wlog(msg,agent,log_file)
+		Wmap::Utils.wlog(msg,agent,log_file)
+	end
+
+	# Host-name mutation for catch easily guessable hostname, i.e. "ww1.example.com" => ["ww1,example.com","ww2.example.com",...]
+	def mutation (host)
+		Wmap::DnsBruter.new.hostname_mutation(host)
+	end
+
+	# Check URL/Site response code
+	def response_code(url)
+		checker=Wmap::UrlChecker.new
+		checker.response_code(url)
+	end
+
+	# Search the site repository for all entries that match the pattern
+	def search(pattern)
+		searcher=Wmap::SiteTracker.new
+		searcher.search(pattern)
+	end
+
+	# Dump out the unique sites into a plain file
+	def dump(file)
+			store=Wmap::SiteTracker.new
+			store.save_uniq_sites(file)
+	end
+
+	# Dump out the unique sites into a XML file
+	def dump_xml(file)
+			store=Wmap::SiteTracker.new
+			store.save_uniq_sites_xml(file)
+	end
+
+	# Refresh the site information in the local data repository
+	def refresh(site)
+			store=Wmap::SiteTracker.new
+			store.refresh(site)
+			store.save!
+	end
+
+	# Refresh the site information in the local data repository
+	def refresh_all
+			store=Wmap::SiteTracker.new
+			store.refresh_all
+			store.save!
+	end
+
+	# Search the Google engines and sort out sites known by Google
+	def google
+		sites=Wmap::GoogleSearchScraper.new.workers.keys
+	end
+
+	# Print a site's full information from the repository
+	def print(site)
+		searcher=Wmap::SiteTracker.new
+		searcher.print_site(site)
+	end
+
+	# Print a site's full information from the repository
+	def print_all
+		searcher=Wmap::SiteTracker.new
+		searcher.print_all_sites
+	end
+
+  private
+
+
+
+  end
+end

data/logs/wmap.log

@@ -0,0 +1,17 @@
+2018-01-10 16:23:21 -0500: wmap: Execute the command: wmap /Users/ylee/www_wmap/uploads/1/seed
+2018-01-10 16:39:40 -0500: wmap: Execute the command: wmap uploads/1/seed
+2018-01-10 16:53:13 -0500: wmap: Execute the command: wmap /Users/ylee/www_wmap/uploads/1/seed
+2018-01-10 17:04:02 -0500: wmap: Execute the command: wmap uploads/1/seed
+2018-01-11 14:33:42 -0500: wmap: Execute the command: wmap uploads/1/seed
+2018-01-11 14:43:00 -0500: wmap: Execute the command: wmap /Users/ylee/www_wmap/uploads/1/seed
+2018-01-11 14:47:02 -0500: wmap: Execute the command: wmap uploads/1/seed
+2018-01-11 14:52:09 -0500: wmap: Execute the command: wmap www.cnn.com
+2018-01-11 15:25:26 -0500: wmap: Execute the command: wmap uploads/1/seed
+2018-01-11 15:25:39 -0500: wdump: Execute the command: wdump 
+2018-01-11 15:25:46 -0500: wdump: Execute the command: wdump /tmp/out
+2018-01-11 15:30:42 -0500: wmap: Execute the command: wmap uploads/1/seed
+2018-01-11 15:46:13 -0500: wmap: Execute the command: wmap uploads/1/seed
+2018-01-11 15:54:04 -0500: wmap: Execute the command: wmap uploads/1/seed
+2018-01-11 15:56:46 -0500: wmap: Execute the command: wmap uploads/1/seed
+2018-01-12 14:04:50 -0500: wmap: Execute the command: wmap /Users/ylee/www_wmap/uploads/1/seed
+2018-01-16 11:25:58 -0500: wmap: Execute the command: wmap 206.156.53.0/24