wmap 2.6.1 → 2.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 95c2c76419e96f7f434f261801b4ba899b1525fd4b9a152f560ab8714cbb8318
-  data.tar.gz: f615e594057de328bd7cafac11b1f8a167d59b8302aeedb77bfa41745aa48590
+  metadata.gz: d327a4a120e75079faa038125baf0f4e39ec784129cd4886a5a81672f0ecf8da
+  data.tar.gz: 8911764a184f1c0f73048433cae9c6848725f3fec470b5d37aab5ce83a0e2789
 SHA512:
-  metadata.gz: 56329a4b3379cf5f79b12521fa5622416bbdc1077a84057f0110ddc6b4a85c6591219221c304b3f69b1cc19ecca291a6312967d66b63702068b991d11c842495
-  data.tar.gz: 308783d83de9ede46f143f9e622f46de6978ad3ef01b8a2047172e21a839e00b18b9286220c6fbd08f623fdc58402f07a961499974c38ae6ed26aaf2cd73aa4c
+  metadata.gz: 25beef23e9ce2771603b4915454e0c40ee6e5b28dc2897c583aecd9e7fbeba2dc07363209f84ec8e596eca36b06222591e197d4a35d187b8140dccee0e16c4c7
+  data.tar.gz: 1a223d84f04fcb1c77f8e2bbb72fa452fd1b3cef076ccce0e80dead367284ce65992125018cf87d9965893d8b418ef3b954898b1244a023305ce75639e447016

data/README.md

@@ -78,7 +78,7 @@ To add your public network block into the scope (note current support of IPv4 on
 
 ### Automatic Discovery and Tracking
 ```sh
-    wmap <seed file | target host | target url | target IP or network cidr>
+    wmap -t <seed file | target host | target url | target IP or network cidr>
 ```
 The above utility is intelligent enough to take argument as either a seed file, or a string such as a host, an IP, a network block, or a URL. The new discoveries will be automatically tracked in the data file 'lib/wmap/data/target_sites'.
   Note: seed file - mix of url, cidr and domain seed, one entry per line.

data/bin/trusts

@@ -12,7 +12,7 @@ Log_dir=File.dirname(__FILE__)+'/../logs/'
 Wmap.wlog("Execute the command: trust #{ARGV[0]}","trust",Log_dir+"wmap.log")
 
 dt=Wmap::DomainTracker.instance
-ct=Wmap::CidrTracker.new(:verbose=>true)
+ct=Wmap::CidrTracker.new(:verbose=>false)
 abort "Incorrect program argument! Proper usage: trust [domain | netblock]" unless ARGV.length==1 && (File.exist?(ARGV[0]))
 
 puts "Start the baptizing process ..."

data/bin/wmap

@@ -2,20 +2,42 @@
 # Wmap main executable - intelligent enough to handle most command argument inputs from the user.
 # The discovery result is automatically compared and saved into the the tracking data repository.
 #
-# Usage: wmap <Target Host | URL | IP | CIDR | or a seed file with any of the above combo> <Optional Discovery Result Directory>
+# Usage: wmap -t <Target Host | URL | IP | CIDR | or a seed file with any of the above combo> -d <Optional Discovery Result Directory>
 require "wmap"
+require "optparse"
 
 # program helper
 def print_usage
-	abort "Program to perform website asset discovery and tracking. \nUsage: wmap <Target Host | URL | IP | CIDR | or a seed file with any of the above combo> <Optional Discovery Result Directory>"
+	abort "Program to perform website asset discovery and tracking. \nUsage: wmap -t <Target Host | URL | IP | CIDR | or a seed file with any of the above combo> -d <Optional Discovery Result Directory>"
 end
+options = {:data_dir => nil, :target => nil}
+
+parser = OptionParser.new do|opts|
+	opts.banner = Wmap.banner
+	opts.on('-d', '--data_dir data_dir', 'Web Mapper local cache data directory') do |data_dir|
+		options[:data_dir] = data_dir;
+	end
+
+	opts.on('-t', '--target target', 'Web Mapper target') do |target|
+		options[:target] = target;
+	end
+
+	opts.on('-h', '--help', 'Displays Help') do
+		Wmap.banner
+		print_usage
+	end
+end
+
+parser.parse!
+
 # print program banner
 puts Wmap.banner
+print_usage unless options[:target]
 
 # Preparing - check out the working logs directory
-if ARGV.length == 2
+if options[:data_dir]
 	# Log to the instance running directory
-	Log_dir = Pathname.new(ARGV[1]).join('logs')
+	Log_dir = Pathname.new(options[:data_dir]).join('logs')
 else
 	# Log the command entry
 	Log_dir=Pathname.new(Gem.loaded_specs['wmap'].full_gem_path).join('logs')
@@ -23,15 +45,14 @@ end
 Dir.mkdir(Log_dir) unless Dir.exist?(Log_dir)
 
 # Start wmap logging
-Wmap.wlog("Execute the command: wmap #{ARGV[0]}","wmap",Log_dir.join("wmap.log").to_s)
-print_usage unless (ARGV.length==1 or ARGV.length==2)
+Wmap.wlog("Execute the command: wmap -t #{options[:target]}","wmap",Log_dir.join("wmap.log").to_s)
 urls = Array.new
 # first step - construct the host list
 scanner = Wmap::PortScanner.new(:verbose=>false, :socket_timeout=>600) # default time-out of 600 milliseconds
 hosts=Array.new
-if File.exist?(ARGV[0])
-	puts "Parsing the discovery seed file: \"#{ARGV[0]}\" "
-	seeds=scanner.file_2_list(ARGV[0])-[nil,""]
+if File.exist?(options[:target])
+	puts "Parsing the discovery seed file: \"#{options[:target]}\" "
+	seeds=scanner.file_2_list(options[:target])-[nil,""]
 	domains=Array.new
 	cidrs=Array.new
 	raise "Error: empty seed file or no legal entry found!" if seeds.nil? or seeds.empty?
@@ -47,22 +68,22 @@ if File.exist?(ARGV[0])
 	puts "Parsing done. "
 	hosts+=Wmap::DnsBruter.new(:verbose=>false).dns_brute_workers(domains.uniq).values.flatten if domains.size > 0
 	cidrs.map { |x| hosts+= scanner.cidr_2_ips(x) } if cidrs.size > 0
-elsif scanner.is_url?(ARGV[0])
-	puts "Processing the URL: #{ARGV[0]}"
-	urls.push(ARGV[0])
-elsif Wmap.domain_known?(ARGV[0]) or Wmap.sub_domain_known?(ARGV[0])
-	puts "Processing the domain: #{ARGV[0]}"
-	hosts+=Wmap::DnsBruter.new(:verbose=>false).dns_brute_worker(ARGV[0]).values.flatten
-elsif scanner.is_fqdn?(ARGV[0])
-	puts "Processing the host: #{ARGV[0]}"
-	hosts.push(ARGV[0])
-	my_hosts=Wmap::DnsBruter.new(:verbose=>false).dns_brute_worker(ARGV[0]).values.flatten if (ARGV[0].split('.')[0] =~ /\d+/)
+elsif scanner.is_url?(options[:target])
+	puts "Processing the URL: #{options[:target]}"
+	urls.push(options[:target])
+elsif Wmap.domain_known?(options[:target]) or Wmap.sub_domain_known?(options[:target])
+	puts "Processing the domain: #{options[:target]}"
+	hosts+=Wmap::DnsBruter.new(:verbose=>false).dns_brute_worker(options[:target]).values.flatten
+elsif scanner.is_fqdn?(options[:target])
+	puts "Processing the host: #{options[:target]}"
+	hosts.push(options[:target])
+	my_hosts=Wmap::DnsBruter.new(:verbose=>false).dns_brute_worker(options[:target]).values.flatten if (options[:target].split('.')[0] =~ /\d+/)
 	hosts+=my_hosts unless my_hosts.nil?
-elsif scanner.is_cidr?(ARGV[0])
-	puts "Processing the network block: #{ARGV[0]}"
-	hosts+=scanner.cidr_2_ips(ARGV[0])
-elsif scanner.is_ip?(ARGV[0])
-	hosts.push(ARGV[0])
+elsif scanner.is_cidr?(options[:target])
+	puts "Processing the network block: #{options[:target]}"
+	hosts+=scanner.cidr_2_ips(options[:target])
+elsif scanner.is_ip?(options[:target])
+	hosts.push(options[:target])
 else
 	print_usage
 end
@@ -77,14 +98,14 @@ scanner=nil
 
 
 # third step - crawling on the URL seeds
-if ARGV.length == 1
+if options[:target] && options[:data_dir]
+	puts "Fire up the crawler with the optional directory setter."
+	crawler = Wmap::UrlCrawler.new(:data_dir => options[:data_dir])
+elsif options[:target]
 	puts "Fire up the crawler."
 	crawler = Wmap::UrlCrawler.new(:verbose=>false)
-elsif ARGV.length == 2
-	puts "Fire up the crawler with the optional directory setter."
-	crawler = Wmap::UrlCrawler.new(:data_dir => ARGV[1])
 else
-	aborts "Error firing up UrlCrawler instance!"
+	abort "Error firing up UrlCrawler instance!"
 end
 Wmap.wlog(urls, "wmap", Log_dir+"url_seeds.log") if urls.size > 0   # save port scan results for debugging
 crawler.crawls(urls) if urls.size>0
@@ -121,17 +142,17 @@ when nil,[]
 	puts "No new site found. There is no change to the site tracking data repository. "
 else
 	puts "Automatically save the discovery results into the site tracking data repository: "
-	if ARGV.length == 1
-		puts "Start the SiteTracker. "
-		inventory=Wmap::SiteTracker.instance
-	elsif ARGV.length == 2
+	if options[:target] && options[:data_dir]
 		puts "Start the SiteTracker with the optional directory setter. "
 		inventory=Wmap::SiteTracker.instance
-		inventory.data_dir = ARGV[1]
+		inventory.data_dir = options[:data_dir]
 		inventory.sites_file = inventory.data_dir + "/" + "sites"
 		inventory.load_site_stores_from_file(inventory.sites_file)
+	elsif options[:target]
+		puts "Start the SiteTracker. "
+		inventory=Wmap::SiteTracker.instance
 	else
-		aborts "Error firing up SiteTracker instance!"
+		abort "Error firing up SiteTracker instance!"
 	end
 	new_sites=inventory.adds(dis_sites.keys)
 	inventory.save! if new_sites.size>0
@@ -141,19 +162,19 @@ end
 
 
 # seventh step - update the hosts repository
-if ARGV.length == 1
-	puts puts "Invoke the HostTracker."
-	host_tracker = Wmap::HostTracker.instance
-	host_tracker.verbose=true
-elsif ARGV.length == 2
+if options[:target] && options[:data_dir]
 	puts "Invoke the HostTracker with optional directory setter."
 	host_tracker = Wmap::HostTracker.instance
-	host_tracker.verbose=true
-	host_tracker.data_dir = ARGV[1]
+	host_tracker.verbose=false
+	host_tracker.data_dir = options[:data_dir]
 	host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
 	host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
+elsif options[:target]
+	puts puts "Invoke the HostTracker."
+	host_tracker = Wmap::HostTracker.instance
+	host_tracker.verbose=false
 else
-	aborts "Error firing up HostTracker instance!"
+	abort "Error firing up HostTracker instance!"
 end
 new_hosts = dis_sites.keys.map {|x| host_tracker.url_2_host(x)}
 hosts += new_hosts

data/lib/wmap/site_tracker.rb

@@ -112,7 +112,7 @@ class Wmap::SiteTracker
 		ip=host_2_ip(host)
 		# Additional logic to refresh deactivated site, 02/12/2014
 		deact=Wmap::SiteTracker::DeactivatedSite.instance
-		deact.sites_file=@data_dir + 'deactivated_sites'
+		deact.sites_file=@data_dir + "/" + "deactivated_sites"
 		File.write(deact.sites_file, "") unless File.exist?(deact.sites_file)
 		deact.load_site_stores_from_file
 		# only trust either the domain or IP we know

data/lib/wmap.rb

@@ -181,7 +181,7 @@ module Wmap
 	# Dump out the unique sites into a plain file
 	def dump(file)
 			store=Wmap::SiteTracker.instance
-      store.verbose=true
+      store.verbose=false
 			store.save_uniq_sites(file)
 	end
 

data/version.txt

@@ -3,8 +3,8 @@
 ###############################################################################
 package = wmap
 # wmap version 2.0 == web_discovery version 1.5.3
-version = 2.6.1
-date = 2019-09-25
+version = 2.6.2
+date = 2019-11-05
 
 author = Sam (Yang) Li
 email = yang.li@owasp.org

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wmap
 version: !ruby/object:Gem::Version
-  version: 2.6.1
+  version: 2.6.2
 platform: ruby
 authors:
 - Sam (Yang) Li
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-25 00:00:00.000000000 Z
+date: 2019-11-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dnsruby