wmap 2.4.4

data/demos/new_fnd.rb

@@ -0,0 +1,66 @@
+###################
+#  Simple lookup function to check if a DMP vuln finding is new found
+#
+#  Usage: ruby new_fnd.rb [old cweid:url pairs] [new cweid.url pair]
+#  				sample input row: "297,https://157.83.142.100/"
+#  Example: ruby new_fnd.rb oldkeys.csv newkeys.csv > new.csv
+
+require "wmap"
+
+@verbose=false
+
+def load_keys (file)
+	puts "Load the key map from file: #{file}" if @verbose
+	host_tracker=Wmap::HostTracker.new
+	my_keys=Hash.new
+	#begin
+		f_old=File.open(file)
+		f_old.each_line do |line|
+			entry=line.chomp.split(',')
+
+			abort "Error loading entry: #{entry}" if entry.size < 2
+			url=entry[1]
+			cveid=entry[0].to_s
+			host=host_tracker.url_2_host(url)
+			ip=String.new
+			if host_tracker.is_fqdn?(host)
+				ip=host_tracker.local_host_2_ip(host)
+			else
+				ip=host
+			end
+			unless host_tracker.is_ip?(ip)
+				ip=host_tracker.host_2_ip(ip)
+			end
+			url.sub!(host,ip) unless ip.nil?
+			entry=cveid+","+url
+			my_keys[entry]=true unless my_keys.key?(entry)
+			puts "Finishing loading key: #{entry}" if @verbose
+		end
+		f_old.close
+		host_tracker=nil
+		return my_keys
+	#rescue => ee
+	#	abort "Error on method #{__method__}: #{ee}" if @verbose
+	#end
+end
+
+old_keys=load_keys(ARGV[0])
+my_tracker=Wmap::HostTracker.new
+f_new=File.open(ARGV[1],'r')
+f_new.each_line do |line|
+	ent=line.chomp.split(',')
+	cve=ent[0]
+	url=ent[1]
+	host=my_tracker.url_2_host(url)
+	ip=my_tracker.local_host_2_ip(host)
+	url.sub!(host,ip) unless ip.nil?
+	entry=cve+","+url
+	if old_keys.key?(entry)
+		puts "#{entry},No"
+	else
+		puts "#{entry},Yes"
+	end
+end
+f_new.close
+my_tracker=nil 
+#puts new_keys.keys.count

data/demos/nmap_parser.pl

@@ -0,0 +1,138 @@
+#!/usr/bin/perl
+########################################################################
+# Functional Description: 
+#                                                                  
+# nmap.pl is designed to be a quick nmap port discovery parser for web_discovery
+#         the program input is NMAP discovery result XML file, the program output
+#         is a list of websites that ready for wmap modules
+#
+#      For usage information, type in the following command:
+# 	$ nmap_parser.pl -h
+# 
+########################################################################
+# Designed and developed by:		Yang Li
+#
+# Change History: 
+# Last modification: 	08/21/2013
+#	Version		0.1
+use Getopt::Long qw/:config bundling_override no_ignore_case/;
+use Nmap::Parser;
+
+my $ver="0.1", $author="Yang Li";				# Program Version and Author
+my $verbose;							# Verbose mode for Maverick
+my %opts;
+GetOptions(
+	\%opts,
+	'help|h|?' => sub { &print_help and exit 0; },		# Print help
+	'version|v:s' => sub { &print_banner; exit 0;},		# Print program version information
+	'file|f:s',                  	 			# Program input file (.xml, .nmap)
+	'output|o:s',               				# Optional, program output result file 
+	'verbose+' => \$verbose,				# Optional, program verbose mode for debugging
+	'vv+' => \$verbose,					# Same as "-verbose", abbreviation "-vv"
+);		
+
+sub parse_nmap_terse () {
+  #
+  ## Parse nmap result in xml format. Only open ports are kept. Web services are further filtered out
+  #	
+	# Check if the optional command switch "-output" is defined, and get ready for it if so.
+	if (defined $opts{output}) { open (OUT, ">", $opts{output}) || die " Can't open the file 8: $opts{output} : $!\n"; }
+	my $np=new Nmap::Parser;
+	$np->parsefile($opts{file});
+	my @HOST=$np->all_hosts("up");						# List of 'up' hosts
+	if (defined $opts{output}) {
+		print OUT "\nTable of Found Open Ports\n";
+		print OUT"IP	Port	Status	Service	OS	Hostname\n";
+		for my $up_host (@HOST){
+			$os = $up_host->os_sig; $osname=$os->name;
+			my $ip=$up_host->addr; my $hostname=$up_host->hostname();
+			print  OUT "$ip\t\t\t\t$osname\t$hostname,\n"; 		# Addr: $up_host->addr, OS: $up_host->os_sig\n";	
+			my @p_tcp=$up_host->tcp_open_ports;
+			my @p_udp=$up_host->udp_open_ports;
+			foreach(@p_tcp) {					# Print list of open tcp ports
+				my $state=$up_host->tcp_port_state($_);
+				my $svc = $up_host->tcp_service($_);
+				my $svc_name = $svc->name;
+				print OUT "\t$_\/tcp\t$state\t$svc_name\n";
+			}
+			foreach(@p_udp) {					# Print list of open udp ports
+				my $state=$up_host->udp_port_state($_);
+				my $svc = $up_host->udp_service($_);
+				my $svc_name = $svc->name;
+				print OUT "\t$_\/udp\t$state\t$svc_name\n";
+			}
+		}
+	} else {		# Redirect to stdout if '-output' command switch is not defined
+		print "List of found web services:\n";
+		#print "IP	Port	Status	Service	OS	Hostname\n";
+		for my $up_host (@HOST){
+			$os = $up_host->os_sig; $osname=$os->name;
+			my $ip=$up_host->addr; my $hostname=$up_host->hostname();
+			#print "$ip\t\t\t\t$osname\t$hostname,\n"; # addr: $up_host->addr, OS: $up_host->os_sig\n";	
+			my @p_tcp=$up_host->tcp_open_ports;
+			my @p_udp=$up_host->udp_open_ports;
+			foreach(@p_tcp) {					# Print list of open tcp ports
+				my $port=$_;
+				my $state=$up_host->tcp_port_state($_);
+				my $svc = $up_host->tcp_service($_);
+				my $svc_name = $svc->name;
+				if ($svc_name =~ /https/i) {
+					if ($hostname) {
+						print "https://$hostname:$port/\n";
+					} else {
+						print "https://$ip:$port/\n";
+					}
+				} elsif ($svc_name =~ /http/i) {
+					if ($hostname) {
+						print "http://$hostname:$port/\n";
+					} else {
+						print "http://$ip:$port/\n"
+					}
+				}
+				#print "\t$_\/tcp\t$state\t$svc_name\n";
+			}
+			foreach(@p_udp) {					# Print list of open udp ports
+				my $port=$_;
+				my $state=$up_host->udp_port_state($_);
+				my $svc = $up_host->udp_service($_);
+				my $svc_name = $svc->name;
+				if ($svc_name =~ /https/i) {
+					if ($hostname) {
+						print "https://$hostname:$port/\n";
+					} else { 
+						print "https://$ip:$port/\n";
+					}
+				} elsif ($svc_name =~ /http/i) {
+					if ($hostname) {
+						print "http://$hostname:$port/\n";
+					} else {
+						print "http://$ip:$port/\n";
+					}
+				}
+				#print "\t$_\/udp\t$state\t$svc_name\n";
+			}	
+		}
+	}
+	if (defined $opts{output}) {
+		close (OUT);  				
+		print "Done dumping out open ports table from $opts{file} to: $opts{output}.\n";
+	}
+	undef $np;
+}
+
+sub print_help {
+		my $header= "#" x 80;
+		print "$header\n nmap result quick parser.\n$header\n";
+		print "Usage: perl nmap.pl -f [nmap xml file] > [result file with found web services]\n";
+		print "Version: $ver, Developed by: $author\n"
+
+}
+
+
+############################################################
+#	Main Program start here
+############################################################
+
+&print_help;
+unless ($opts{file}) { print "Error: unknown program input. Please check your file again. \n"; exit 1; }
+parse_nmap_terse ();

data/demos/site_format.rb

@@ -0,0 +1,18 @@
+# filter to detect unknown internet domain
+# Input is a list of URLs
+# Output is an internet domain list that not currently tracked by the domain tracker
+
+require "wmap"
+
+k=Wmap::SiteTracker.new
+f=File.open(ARGV[0],'r')
+f.each do |line|
+	url=line.chomp.strip.downcase
+	if k.is_url?(url)
+		puts k.url_2_site(url)
+	else
+		puts url
+	end
+end
+f.close
+k=nil 

data/demos/whois_domain.rb

@@ -0,0 +1,78 @@
+# Automated whois lookup for a list of valid domains
+# Usage: ruby whois_domain.rb [file_hosts]
+# Input file format: one line for each host (FQDN)
+# Output file format: good old CSV format, with whois response parsed and sorted into structured fields.
+require	"wmap"
+
+puts Wmap.banner
+dis=Wmap::DomainTracker.new
+dis.verbose=false
+puts "Domain Whois Lookup Summary Report"
+puts "Host | Domain | Primary Domain Name Server | Registrant Name | Registrant Oraganization | Registrant Address | Registrant Zip | Registrant City | Registrant State | Registration Country | Registration Contact Phone | Registration Contact Email | Technical Contact Name | Technical Contact Organization | Technical Contact Country | Technical Contract Phone | Technical Contact Email | Admin Contact Name | Admin Contact Organization | Domain Availability"
+f_hosts = File.open(ARGV[0], 'r')
+f_hosts.each do |line|
+	#next if line.chomp =~ /\d+\.\d+\.\d+\.\d+/
+	begin
+		host=line.chomp.split(',')[0]
+		#url=line.chomp.split(',')[0]
+		#host=dis.url_2_host(url)
+		domain=dis.domain_root(host)
+		record=Hash.new
+		if domain.nil?
+			puts "#{line.chomp} | Domain Unknown"
+#		elsif dis.domain_known?(domain)
+#			next
+		else
+			result=Wmap.whois(domain)
+			puts result if dis.verbose
+			record['ns'] = result.nameservers.first unless result.nameservers.nil?
+			unless result.registrant_contacts.nil?
+				result.registrant_contacts.each do |contact_r|
+					 record['r_name']=contact_r['name']
+					 record['r_org']=contact_r['organization']
+					 record['r_addr']=contact_r['address'].gsub(/\n/,',').gsub(/\r/,' ') unless contact_r['address'].nil?
+					 record['r_zip']=contact_r['zip']
+					 record['r_city']=contact_r['city']
+					 record['r_state']=contact_r['state']
+					 record['r_country']=contact_r['country_code']
+					 record['r_phone']=contact_r['phone']
+					 record['r_email']=contact_r['email']
+				end
+			end
+			unless result.technical_contacts.nil?
+				result.technical_contacts.each do |contact_t|
+					 record['t_name']=contact_t['name']
+					 record['t_org']=contact_t['organization']
+					 record['t_country']=contact_t['country_code']
+					 record['t_phone']=contact_t['phone']
+					 record['t_email']=contact_t['email']
+				end
+			end
+			unless result.admin_contacts.nil?
+				result.admin_contacts.each do |contact_a|
+					 record['a_name']=contact_a['name']
+					 record['a_org']=contact_a['organization']
+				end
+			end
+			if result.available?
+				record['availability']="true"
+			else
+				record['availability']="false"
+			end
+		end
+		print "#{line.chomp} | #{domain} | "
+		if record.nil?
+			print " | Failure to parse the whois response. Please add it manually. "
+		else
+			print record['ns'], '|'
+			print record['r_name'], ' | ', record['r_org'],' | ', record['r_addr'],' | ', record['r_zip'],' | ', record['r_city'],' | ', record['r_state'],' | ', record['r_country'],' | ', record['r_phone'],' | ', record['r_email']
+			print ' | ', record['t_name'], ' | ', record['t_org'],' | ', record['t_country'],' | ', record['t_phone'],' | ', record['t_email']
+			print ' | ', record['a_name'], ' | ', record['a_org'],' | ', record['availability']
+		end
+		print "\n"
+	rescue => ee
+		puts "#{line.chomp} | #{domain} | #{ee}"
+	end
+end
+f_hosts.close
+dis=nil