wmap 2.6.7 → 2.6.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e4f2a2dfe9b4b119331eefffc7b9b025d9953c2ce5f7255e4d2a08929a591c3
-  data.tar.gz: 3d018d69469cf4e4551b38397657341661fd95c3f59bebe8bb21405d4e107881
+  metadata.gz: 5cbae048aecd25ee91bdbbf75657d8f4329e9abd2c65cd90cedd65b553659ed4
+  data.tar.gz: 1043ec8e2dd15287c6d11b3a0094ca4b65c4684aec96cdea03085cfea431ceb8
 SHA512:
-  metadata.gz: 7e7d27b4d4abfc34ab3df0933412b4c99e94af93f71251a2e7a0706b4782ad62a2541dbf7c5f391f47d5a3b9eae9feb45ce1690b2e43fddab4f52a14e7bb334b
-  data.tar.gz: 0117422a9eac9f1c7a66783a0a4ca870711aeaa95252a9df8fe4f9ce2f8f10fd4ba461b2146a681f3b867becb9bace05c61a3e11b284714732432c95e12dc983
+  metadata.gz: b89c65b1ec441b0307ef85a322f8ced64e9443694a021301989a52761cc62496e3c2552fb21c709a01f1bf8e7834634f8d2e4d9ce6432521c6a197e02a9f7290
+  data.tar.gz: 71b0beedf0e8c9f75172cde52c6f40b9620a96de5ecfc69c031908e6c86534818ca1f5208711a3eb9c5852a09c1b2beda864d25c620399737442e65016bcbf98

data/README.md

@@ -15,12 +15,13 @@
 
 
 ## What's this program for?
-This program is designed for the web application asset discovery and tracking. It was originally developed to
-to cover the gaps of a similar commercial product. Over the time it grows to be a more capable and complete replacement (IMHO).
+This program is part of the [OWASP Web Mapper Project](https://www.owasp.org/index.php/OWASP_Web_Mapper_Project). It's designed for the web application asset discovery and tracking. It was originally developed to cover the gaps of a similar commercial offering. Over the time it grows to be a more capable and complete replacement (IMHO).
+
+Note that program is mainly operating on Command Line Interface (CLI). For better user experience, you might want to use the [Web Mapper Portal Application](https://github.com/yangsec888/www_wmap) instead.
 
 
 ## WMAP in Motion
-Use the demo web app build on top of wmap gem: http://wmap.io
+You can try out the complete [demo web app](http://wmap.io/) deployed in the DigitalOcean cloud: http://wmap.io/
 
 
 ## Installation
@@ -35,7 +36,8 @@ Nokogiri is a native xml/html parser used by the project. It's fast and powerful
 ### Dependency
 You need the Ruby 2.1.0 or above in order to use this program. In my test environment, I was able to set it up with <a href="https://rvm.io/">RVM</a>. Please refer to this page for more installation information: https://www.ruby-lang.org/en/documentation/installation/
 
-In addition, the following Ruby GEM dependency are needed by different components of this software. The should be installed automatically:
+In addition, the following Ruby GEM dependency are needed by different features of this software. They should be installed automatically when you install the 'wmap' gem above.
+```
       require "dnsruby"
       require "geoip"
       require "minitest/autorun"
@@ -48,11 +50,13 @@ In addition, the following Ruby GEM dependency are needed by different component
       require "parallel"
       require "whois"
       require 'httpclient'
-
+```
 
 In case you want to install the above gems separately, use the command below:
-
+```sh
       gem install dnsruby geoip minitest net-ping netaddr nokogiri css_parser open_uri_redirections openssl parallel whois httpclient
+```
+
 
 ### Ruby-whois Gem Patches
 This software depends on a patched version of Ruby gem ruby-whois (http://www.ruby-whois.org/) for the domain whois lookup feature. For better result, you could manually add the patches into your local whois gem installation directory as shown below:
@@ -106,7 +110,7 @@ If you need additional documentation / information other than this README file a
 
 
 ## Program Version
-The latest release is version [2.5.5+](version.txt). as of fall 2018. Please refer to the [CHANGELOG.md](CHANGELOG.md) for more history information.
+The latest release is version [2.6.5+](version.txt). as of fall 2019. Please refer to the [CHANGELOG.md](CHANGELOG.md) for more history information.
 
 
 ## Author Contact

data/bin/updateAll

@@ -1,14 +1,41 @@
 #!/usr/bin/env ruby
 # the up to date program to refresh all local cache tables in one shot. Note it requires an uninterrupted Internet connection to perform the job. It also takes a long time so patient is needed. Lastly, don't forget to backup the existing 'data' folder before execute this command, cause any unexpected interruption may wreak havoc on the existing data file!!!
-
+#
+#
+# Usage: updateAll -d <Optional Data Directory>
 require "wmap"
+require "optparse"
+
+# program command line options
+options = {:data_dir => nil, :target => nil, :verbose => false}
+parser = OptionParser.new do|opts|
+	opts.banner = Wmap.banner
+	opts.on('-d', '--data_dir data_dir', 'Web Mapper local cache data directory') do |data_dir|
+		options[:data_dir] = data_dir;
+	end
+	opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
+		options[:verbose] = v;
+	end
+	opts.on('-h', '--help', 'Displays Help') do
+		puts opts
+		exit 0
+	end
+end
+parser.parse!
 
 puts Wmap.banner
 puts "Program to refresh the local data repository."
-Log_dir=File.dirname(__FILE__)+'/../logs/'
+# Preparing - check out the working logs directory
+if options[:data_dir]
+	# Log to the instance running directory
+	Log_dir = Pathname.new(options[:data_dir]).join('logs')
+else
+	# Log the command entry
+	Log_dir=Pathname.new(Gem.loaded_specs['wmap'].full_gem_path).join('logs')
+end
+Dir.mkdir(Log_dir) unless Dir.exist?(Log_dir)
 Wmap.wlog("Execute the command: updateAll","updateAll",Log_dir+"wmap.log")
 
-abort "Incorrect program argument - no argument needed! Proper Usage: updateAll" unless ARGV.length==0
 =begin
 puts "You're about to update Wmap data repository. It'll take a long time. And the Internet connection must be un-interrupted during the process. You're also expected to backup the data folder before proceeding. Are you ready? (Yes/No)"
 STDOUT.flush
@@ -21,22 +48,41 @@ end
 =end
 # Update sub-domain table
 sd=Wmap::DomainTracker::SubDomain.instance
+if options[:data_dir]
+  sd.data_dir=options[:data_dir]
+  sd.sub_domains_file = sd.data_dir + "/" + "sub_domains"
+  sd.known_internet_sub_domains = sd.load_domains_from_file(sd.sub_domains_file)
+end
 sd.update_from_host_store!
 subs=sd.known_internet_sub_domains.keys
 sd=nil
 
 # Update Domain table
 dm=Wmap::DomainTracker.instance
+if options[:data_dir]
+  dm.data_dir=options[:data_dir]
+  dm.domains_file=dm.data_dir + "/" + "domains"
+  dm.load_domains_from_file(dm.domains_file)
+end
 domains=dm.known_internet_domains.keys
 dm=nil
 
 # Brute force sub-domains to detect sub-domain hosts
-bruter=Wmap::DnsBruter.new
+bruter=Wmap::DnsBruter.new(:verbose => options[:verbose])
+if options[:data_dir]
+  bruter.data_dir=options[:data_dir]
+  bruter.file_hosts = bruter.data_dir + "/" + "hosts"
+end
 sub_hosts=bruter.brutes(subs).values.flatten
 hosts=bruter.brutes(domains).values.flatten
 
 # Update primary host store
 ph=Wmap::HostTracker::PrimaryHost.instance
+if options[:data_dir]
+  ph.data_dir=options[:data_dir]
+  ph.hosts_file = ph.data_dir + "/" + "prime_hosts"
+  ph.load_known_hosts_from_file(ph.hosts_file)
+end
 ph.update_from_site_store!
 ph.refresh_all
 ph.save!
@@ -44,6 +90,11 @@ ph=nil
 
 # Update host store
 h=Wmap::HostTracker.instance
+if options[:data_dir]
+  h.data_dir=options[:data_dir]
+  h.hosts_file=h.data_dir + "/" +"hosts"
+  h.load_known_hosts_from_file(h.hosts_file)
+end
 h.refresh_all
 h.adds(sub_hosts)
 h.adds(hosts)
@@ -52,10 +103,21 @@ h=nil
 
 # Update site store
 st=Wmap::SiteTracker.instance
+if options[:data_dir]
+  st.data_dir=options[:data_dir]
+  st.sites_file=st.data_dir + "/" +"sites"
+  st.load_site_stores_from_file(st.sites_file)
+end
 st.refresh_all
+# double-check the de-activated sites in case the site is back on-line again
 dt=Wmap::SiteTracker::DeactivatedSite.instance
+if options[:data_dir]
+  dt.data_dir = options[:data_dir]
+  dt.sites_file = dt.data_dir + "/" + "deactivated_sites"
+  dt.known_sites = dt.load_site_stores_from_file(dt.sites_file)
+end
 ds=dt.known_sites.keys
-st.adds(ds)					#double-check the de-activated sites in case the site is back on-line again
+st.adds(ds)
 st.save!
 st=nil
 dt=nil

data/bin/wmap

@@ -28,7 +28,7 @@ parser.parse!
 
 # print program banner
 puts Wmap.banner
-print_usage unless options[:target]
+# print_usage unless options[:target]
 
 # Preparing - check out the working logs directory
 if options[:data_dir]

data/lib/wmap/dns_bruter.rb

@@ -13,7 +13,7 @@ require "parallel"
 class Wmap::DnsBruter
 	include Wmap::Utils
 
-	attr_accessor :hosts_dict, :verbose, :max_parallel, :data_dir
+	attr_accessor :hosts_dict, :verbose, :max_parallel, :data_dir, :file_hosts
 	attr_reader :discovered_hosts_from_dns_bruter, :fail_domain_cnt
 
 	# Set default instance variables
@@ -22,11 +22,9 @@ class Wmap::DnsBruter
 		@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
 		Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
 		@file_hosts = @data_dir + 'hosts'
-		@file_hosts_dict = File.dirname(__FILE__)+'/../../dicts/hostnames-dict.txt'
-
 		@verbose=params.fetch(:verbose, false)
 		@discovered_hosts_from_dns_bruter=Hash.new
-		@hosts_dict=params.fetch(:hosts_dict, @file_hosts_dict)
+		@hosts_dict=params.fetch(:hosts_dict, File.dirname(__FILE__)+'/../../dicts/hostnames-dict.txt')
 		@max_parallel=params.fetch(:max_parallel, 30)
 		@fail_domain_cnt=Hash.new
 	end

data/lib/wmap/domain_tracker/sub_domain.rb

@@ -23,6 +23,7 @@ class SubDomain < Wmap::DomainTracker
 	def initialize (params = {})
 		@verbose=params.fetch(:verbose, false)
 		@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../../data/')
+		Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
 		@max_parallel=params.fetch(:max_parallel, 40)
 		# Hash table to hold the trusted domains
 		@sub_domains_file=params.fetch(:sub_domains_file, @data_dir + 'sub_domains')
@@ -85,7 +86,9 @@ class SubDomain < Wmap::DomainTracker
 		puts "Invoke internal procedures to update the sub-domain list from the host store."
 		# Step 1 - obtain the latest sub-domains
 		my_tracker = Wmap::HostTracker.instance
-		my_tracker.data_dir=@data_dir
+		my_tracker.data_dir = @data_dir
+		my_tracker.hosts_file = my_tracker.data_dir + "/" + "hosts"
+		my_tracker.load_known_hosts_from_file(my_tracker.hosts_file)
 		subs = my_tracker.dump_sub_domains - [nil,""]
 		my_tracker = nil
 		# Step 2 - update the sub-domain list
@@ -133,4 +136,4 @@ class SubDomain < Wmap::DomainTracker
 
 end
 end
-end 
+end

data/lib/wmap/site_tracker.rb

@@ -136,6 +136,7 @@ class Wmap::SiteTracker
 			# Add logic to check site status before adding it
 			checker=Wmap::UrlChecker.new(:data_dir=>@data_dir).check(site)
 			raise "Site is currently down. Skip #{site}" if checker.nil?
+			raise "Site is time-out. Skip #{site}" if checker["code"] == 10000
 			# Skip the http site if it's un-responsive; for the https we'll keep it because we're interested in analysing the SSL layer later
 			if is_https?(site)
 				# do nothing

data/lib/wmap/wp_tracker.rb

@@ -36,7 +36,7 @@ class Wmap::WpTracker
 	end
 
   # 'setter' to load the known wordpress sites into an instance variable
-	def load_from_file (file=@file_stores, lc=true)
+	def load_from_file (file=@sites_wp, lc=true)
 		puts "Loading trusted file: #{file}"	if @verbose
 		@known_wp_sites=Hash.new
 		f_wp_sites=File.open(file, 'r')

data/version.txt

@@ -3,8 +3,8 @@
 ###############################################################################
 package = wmap
 # wmap version 2.0 == web_discovery version 1.5.3
-version = 2.6.7
-date = 2019-11-19
+version = 2.6.8
+date = 2019-11-15
 
 author = Sam (Yang) Li
 email = yang.li@owasp.org

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wmap
 version: !ruby/object:Gem::Version
-  version: 2.6.7
+  version: 2.6.8
 platform: ruby
 authors:
 - Sam (Yang) Li
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-19 00:00:00.000000000 Z
+date: 2019-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dnsruby