wmap 2.7.2 → 2.7.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ede228cdbc1b9ce2e12369c1d5c61fba8549f8c8dc67c35ee009020cfc32c014
-  data.tar.gz: 06ea77a19cbbc5b2bc72aea90fefaff6c336ee1b75a8eba765730e660751426f
+  metadata.gz: f1f2c900ee2ac330d411f3f6496868ea519dd322b236cd00be0b5f3d1d8d43db
+  data.tar.gz: 054a534426e0b9b99730ac6252a9b0e2424866df914d02f6f7f1bdb925588bc1
 SHA512:
-  metadata.gz: 22ce85d7d400650fb8427518c9ce79568db1143178dc60dcedbaa8683f5f4f58a461ebd4065fc078e3ff05e7cb80c7a016b08638dc793dc1150dfcce101de4fb
-  data.tar.gz: 3b51d6bda550f065a675409334b519be76521ab3014956b8f017e07d0be4fc33ab41d2b7ddb2e6dcc552fe9d36ce68cb8a887b0895b28470c7b553b18620a1e3
+  metadata.gz: c591b085614ec050ff8637750dfff57c3238ccf42c2fce9cab985ccd0f3e4761ed085632a9db7f4af87be39e2921b33d330c890c1af87213a134bdd37f43553d
+  data.tar.gz: 414720066fee25c30b41beb9c7529f7bfffda513e01e1bf4445cfefcd9ecbf2c421aa64db931a4d489845d664705ededb2474c3240750ad12cd2423b21078987

data/README.md

@@ -27,7 +27,7 @@ You can try out the complete [demo web app](http://wmap.io/) deployed in the Dig
 ## Installation
 To take full power of this program, you would need an *nix flavor machine with direct Internet access. I have installed it successfully on both Mac and Linux machines. You'll also need the Ruby environment being setup properly. The easiest way to install OWASP Web Mapper is by using Ruby Gems. You can install it from command line:
 ```sh
-  gem install wmap
+$ gem install wmap
 ```
 
 ### Specific Installation Problem with Nokogiri
@@ -54,30 +54,24 @@ In addition, the following Ruby GEM dependency are needed by different features
 
 In case you want to install the above gems separately, use the command below:
 ```sh
-      gem install dnsruby geoip minitest net-ping netaddr nokogiri css_parser open_uri_redirections openssl parallel whois httpclient
+$ gem install dnsruby geoip minitest net-ping netaddr nokogiri css_parser open_uri_redirections openssl parallel whois httpclient
 ```
 
-
-### Ruby-whois Gem Patches
-This software depends on a patched version of Ruby gem ruby-whois (http://www.ruby-whois.org/) for the domain whois lookup feature. For better result, you could manually add the patches into your local whois gem installation directory as shown below:
-```sh
-      cp whois_patches/* [Your_ruby_whois_gem_path]/whois/lib/whois/record/parser/
-```
-
-Or you can directly download the branched whois gem from this repository - https://github.com/yangsec888/whois
-
-
 ### Before Using This Program
 You need to define a scope for the program to run successful. The scope includes both your legitimate Internet domain, and your public
 network block in the CIDR format.
 
 To add your Internet domain into the scope, use the build-in shell command below:
 ```sh
-     trust XYZ.COM
+$ trust --target XYZ.COM
 ```
 To add your public network block into the scope (note current support of IPv4 only):
 ```sh
-     trust x.x.x.x/x
+$ trust --target x.x.x.x/x
+```
+Use command switch '--help' for more information
+```sh
+$ trust --help
 ```
 
 ### Automatic Discovery and Tracking
@@ -94,7 +88,7 @@ The above utility is intelligent enough to take argument as either a seed file,
 ### Dump Out Discovery Database
 You can dump out the program output by using the build-in utility 'wdump' as shown below:
 ```sh
-    wdump [output file name from you]
+$ wdump [output file name from you]
 ```
 The above utility will dump out the discovery database into a single file as program output. Currently, the supported file format is Comma-separated Value (.csv) and Extensible Markup Language (.xml)
 
@@ -114,7 +108,7 @@ The latest release is version [2.6.5+](version.txt). as of fall 2019. Please ref
 
 
 ## Author Contact
-This program is designed and developed by Yang Li. You can reach him by Email: <yang.li@owasp.org>
+This program is designed and developed by Sam (Yang) Li. You can reach him by Email: <yang.li@owasp.org>
 ## Bug Report or Feature Request?
 Contact the author Sam (Yang) Li directly at email 'yang.li@owasp.org'.
 

data/bin/refresh

@@ -37,11 +37,11 @@ store.verbose = true
 # Preparing - check out the working logs directory
 if options[:data_dir]
 	# Log to the instance running directory
-	Log_dir = Pathname.new(options[:data_dir]).join('logs')
+	Log_dir = Pathname.new(options[:data_dir]).join('/../logs')
 	store.data_dir = options[:data_dir]
-	sites_file = options[:data_dir] + '/' + 'sites'
+	store.sites_file = options[:data_dir] + '/' + 'sites'
 	#puts "sites_file: #{sites_file}"
-	store.load_site_stores_from_file(sites_file)
+	store.load_site_stores_from_file(store.sites_file)
 else
 	# Log the command entry
 	Log_dir=Pathname.new(Gem.loaded_specs['wmap'].full_gem_path).join('logs')
@@ -50,7 +50,7 @@ end
 puts Wmap.banner
 print_usage
 # Log_dir=File.dirname(__FILE__)+'/../logs/'
-Wmap.wlog("Execute the command: refresh #{ARGV[0]}","refresh",Log_dir+"wmap.log")
+Wmap.wlog("Execute the command: refresh #{ARGV[0]}","refresh",Log_dir.to_s+"wmap.log")
 
 #puts "Captured command argument"
 if options[:target]
@@ -66,7 +66,7 @@ end
 
 # please
 if options[:data_dir]
-	store.save!(sites_file)
+	store.save!(store.sites_file)
 else
 	store.save!
 end

data/lib/wmap.rb

@@ -16,7 +16,6 @@ require 'wmap/host_tracker'
 require 'wmap/host_tracker/primary_host'
 require 'wmap/whois'
 require 'wmap/url_checker'
-require 'wmap/wp_tracker'
 require 'wmap/network_profiler'
 require 'wmap/port_scanner'
 require 'wmap/url_crawler'
@@ -24,6 +23,7 @@ require 'wmap/url_crawler/adware_tag'
 require 'wmap/dns_bruter'
 require 'wmap/site_tracker'
 require 'wmap/site_tracker/deactivated_site'
+require 'wmap/site_tracker/wp_tracker'
 require 'wmap/geoip_tracker'
 require 'wmap/google_search_scraper'
 

data/lib/wmap/site_tracker.rb

@@ -189,13 +189,11 @@ class Wmap::SiteTracker
 						old_ip=host_tracker.local_host_2_ip(host)
 						if old_ip != ip
 							host_tracker.refresh(host)
-							host_tracker.save!
 						else
 							puts "Host resolve to the same IP #{ip} - no need to update the local host table." if @verbose
 						end
 					else
 						host_tracker.add(host)
-						host_tracker.save!
 					end
 				end
 			else
@@ -214,13 +212,11 @@ class Wmap::SiteTracker
 					old_ip=host_tracker.local_host_2_ip(host)
 					if old_ip != ip
 						host_tracker.refresh(host)
-						host_tracker.save!
 					else
 						# Skip - no need to update the local hosts table
 					end
 				else
 					host_tracker.add(host)
-					host_tracker.save!
 				end
 			end
 			deact=nil
@@ -229,6 +225,7 @@ class Wmap::SiteTracker
 		else
 			puts "Problem found: untrusted Internet domain or IP. Skip #{site}"
 			deact=nil
+			host_tracker.save!
 			host_tracker=nil
 			return nil
 		end

data/lib/wmap/{wp_tracker.rb → site_tracker/wp_tracker.rb}

@@ -12,9 +12,10 @@ require "open_uri_redirections"
 require "nokogiri"
 require "css_parser"
 
+module Wmap
+class SiteTracker
 
-# Main class to automatically track the site inventory
-class Wmap::WpTracker
+class WpTracker < Wmap::SiteTracker
 	include Wmap::Utils
 	#include Singleton
 
@@ -24,7 +25,7 @@ class Wmap::WpTracker
   # WordPress checker instance default variables
 	def initialize (params = {})
 		@verbose=params.fetch(:verbose, false)
-		@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
+		@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../../data/')
 		Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
     @sites_wp=params.fetch(:sites_wp, @data_dir+"wp_sites")
 		@http_timeout=params.fetch(:http_timeout, 5000)
@@ -116,6 +117,41 @@ class Wmap::WpTracker
 		puts "Exception on method #{__method__}: #{ee}: #{url}" if @verbose
 	end
 
+	# Method to load wp sites in parallel
+	def bulk_add(list,num=@max_parallel,use_cache=true)
+		puts "Add entries to the local wp_site store from list:\n #{list}"
+		results=Hash.new
+		list = list - [nil,""]
+		if list.size > 0
+			puts "Start parallel adding on the sites:\n #{list}"
+			Parallel.map(list, :in_processes => num) { |target|
+				add(target,use_cache)
+			}.each do |process|
+				if process.nil?
+					next
+				elsif process.empty?
+					next #do nothing
+				else
+					results[process['site']]=Hash.new
+					results[process['site']]=process
+				end
+			end
+			@known_wp_sites.merge!(results)
+		else
+			puts "Error: no entry is added. Please check your list and try again."
+		end
+		puts "Done adding site entries."
+		if results.size>0
+			puts "New entries added: #{results}"
+		else
+			puts "No new entry added. "
+		end
+		return results
+	rescue => ee
+		puts "Exception on method #{__method__}: #{ee}" if @verbose
+	end
+	alias_method :adds, :bulk_add
+
   # logic to determin if it's a wordpress site
   def is_wp?(url)
 		site=url_2_site(url)
@@ -375,5 +411,6 @@ class Wmap::WpTracker
 		return nil
 	end
 
-
+end
+end
 end

data/lib/wmap/url_crawler.rb

@@ -41,7 +41,7 @@ class Wmap::UrlCrawler
 		@crawl_start=Hash.new
 		@crawl_done=Hash.new
 		Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
-		@log_file=@data_dir + "crawler.log"
+		@log_file=@data_dir + "/../logs/crawler.log"
 	end
 
 	# Pre-crawl profiler, to be used for network profiling to maximum the crawler performance.

data/version.txt

@@ -3,8 +3,8 @@
 ###############################################################################
 package = wmap
 # wmap version 2.0 == web_discovery version 1.5.3
-version = 2.7.2
-date = 2020-03-09
+version = 2.7.6
+date = 2020-03-20
 
 author = Sam (Yang) Li
 email = yang.li@owasp.org

data/wmap.gemspec

@@ -35,10 +35,10 @@ Gem::Specification.new do |s|
   s.date = info["date"]
   s.description = "wmap is written to perform Internet web application / service discovery. The discovery results are designed to be automatically tracked by the software."
   s.email = info["email"]
-  s.executables = ["wmap","wscan","wadd","wadds","wdel","wcheck","wdump","spiderBot","googleBot","updateAll","prime","deprime","refresh","trust","distrust","run_tests"]
+  s.executables = ["wmap","wscan","wadd","wadds","wdel","wcheck","wdump","spiderBot","googleBot","updateAll","prime","deprime","refresh","trust","trusts","distrust","run_tests"]
   s.files = ["CHANGELOG.md", "TODO", "settings/discovery_ports","data/","LICENSE.txt",
 							"version.txt","README.md", "wmap.gemspec"]
-  s.files += Dir['lib/*.rb'] + Dir['lib/wmap/*.rb'] + Dir['lib/wmap/**/*'] + Dir['bin/*'] + Dir['settings/*'] + Dir['demos/*'] + Dir['test/*'] + Dir['ruby_whois_patches/*'] + Dir['dicts/*']
+  s.files += Dir['lib/*.rb'] + Dir['lib/wmap/*.rb'] + Dir['lib/wmap/**/*'] + Dir['bin/*'] + Dir['settings/*'] + Dir['demos/*'] + Dir['test/*'] + Dir['dicts/*']
   #s.homepage = "none"
   s.post_install_message = "*"*80 + "\n\nThank you for installing the wmap gem - a pure Ruby library for Internet web \napplication discovery and tracking. Please refer to the README.md for more \ninformation of using this gem.  \n\n" + "*"*80 + "\n"
   s.require_paths = ["lib"]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wmap
 version: !ruby/object:Gem::Version
-  version: 2.7.2
+  version: 2.7.6
 platform: ruby
 authors:
 - Sam (Yang) Li
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-09 00:00:00.000000000 Z
+date: 2020-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dnsruby
@@ -224,6 +224,7 @@ executables:
 - deprime
 - refresh
 - trust
+- trusts
 - distrust
 - run_tests
 extensions: []
@@ -294,6 +295,7 @@ files:
 - lib/wmap/port_scanner.rb
 - lib/wmap/site_tracker.rb
 - lib/wmap/site_tracker/deactivated_site.rb
+- lib/wmap/site_tracker/wp_tracker.rb
 - lib/wmap/url_checker.rb
 - lib/wmap/url_crawler.rb
 - lib/wmap/url_crawler/adware_tag.rb
@@ -302,63 +304,6 @@ files:
 - lib/wmap/utils/url_magic.rb
 - lib/wmap/utils/utils.rb
 - lib/wmap/whois.rb
-- lib/wmap/wp_tracker.rb
-- ruby_whois_patches/base_cocca2.rb
-- ruby_whois_patches/kero.yachay.pe.rb
-- ruby_whois_patches/whois.PublicDomainRegistry.com.rb
-- ruby_whois_patches/whois.above.com.rb
-- ruby_whois_patches/whois.adamsnames.tc.rb
-- ruby_whois_patches/whois.aeda.net.ae.rb
-- ruby_whois_patches/whois.ai.rb
-- ruby_whois_patches/whois.arnes.si.rb
-- ruby_whois_patches/whois.ascio.com.rb
-- ruby_whois_patches/whois.cnnic.cn.rb
-- ruby_whois_patches/whois.corporatedomains.com.rb
-- ruby_whois_patches/whois.crsnic.net.rb
-- ruby_whois_patches/whois.denic.de.rb
-- ruby_whois_patches/whois.dk-hostmaster.dk.rb
-- ruby_whois_patches/whois.dns.be.rb
-- ruby_whois_patches/whois.dns.lu.rb
-- ruby_whois_patches/whois.dns.pl.rb
-- ruby_whois_patches/whois.dns.pt.rb
-- ruby_whois_patches/whois.domain.kg.rb
-- ruby_whois_patches/whois.domainregistry.my.rb
-- ruby_whois_patches/whois.domreg.lt.rb
-- ruby_whois_patches/whois.dot.tk.rb
-- ruby_whois_patches/whois.hkirc.hk.rb
-- ruby_whois_patches/whois.isnic.is.rb
-- ruby_whois_patches/whois.je.rb
-- ruby_whois_patches/whois.jprs.jp.rb
-- ruby_whois_patches/whois.kenic.or.ke.rb
-- ruby_whois_patches/whois.markmonitor.com.rb
-- ruby_whois_patches/whois.melbourneit.com.rb
-- ruby_whois_patches/whois.nic.as.rb
-- ruby_whois_patches/whois.nic.at.rb
-- ruby_whois_patches/whois.nic.ch.rb
-- ruby_whois_patches/whois.nic.cl.rb
-- ruby_whois_patches/whois.nic.ec.rb
-- ruby_whois_patches/whois.nic.im.rb
-- ruby_whois_patches/whois.nic.it.rb
-- ruby_whois_patches/whois.nic.lv.rb
-- ruby_whois_patches/whois.nic.ly.rb
-- ruby_whois_patches/whois.nic.mu.rb
-- ruby_whois_patches/whois.nic.mx.rb
-- ruby_whois_patches/whois.nic.net.sa.rb
-- ruby_whois_patches/whois.nic.or.kr.rb
-- ruby_whois_patches/whois.nic.tel.rb
-- ruby_whois_patches/whois.nic.tr.rb
-- ruby_whois_patches/whois.nic.us.rb
-- ruby_whois_patches/whois.nic.ve.rb
-- ruby_whois_patches/whois.norid.no.rb
-- ruby_whois_patches/whois.pandi.or.id.rb
-- ruby_whois_patches/whois.psi-usa.info.rb
-- ruby_whois_patches/whois.registro.br.rb
-- ruby_whois_patches/whois.registrygate.com.rb
-- ruby_whois_patches/whois.rrpproxy.net.rb
-- ruby_whois_patches/whois.sgnic.sg.rb
-- ruby_whois_patches/whois.srs.net.nz.rb
-- ruby_whois_patches/whois.tucows.com.rb
-- ruby_whois_patches/whois.twnic.net.tw.rb
 - settings/discovery_ports
 - settings/google_keywords.txt
 - settings/google_locator.txt

data/ruby_whois_patches/base_cocca2.rb

@@ -1,149 +0,0 @@
-#--
-# Ruby Whois
-#
-# An intelligent pure Ruby WHOIS client and parser.
-#
-# Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
-#++
-
-
-require 'whois/record/parser/base'
-
-
-module Whois
-  class Record
-    class Parser
-
-      # Base parser for CoCCA servers.
-      #
-      # @abstract
-      class BaseCocca2 < Base
-
-        property_supported :domain do
-          content_for_scanner =~ /Domain Name: (.+)\n/
-          $1 || Whois.bug!(ParserError, "Unable to parse domain.")
-        end
-
-        property_not_supported :domain_id
-
-        # TODO: /pending delete/ => :redemption
-        # TODO: /pending purge/  => :redemption
-        property_supported :status do
-          list = statuses
-          case
-            when list.empty?
-              Whois.bug!(ParserError, "Unable to parse status.")
-            when list.include?("available")
-              :available
-            when list.include?("ok")
-              :registered
-            else
-              Whois.bug!(ParserError, "Unknown status `#{list.join(", ")}'.")
-          end
-        end
-
-        property_supported :available? do
-          status == :available
-        end
-
-        property_supported :registered? do
-          !available?
-        end
-
-
-        property_supported :created_on do
-          if content_for_scanner =~ /Creation Date: (.+?)\n/
-            parse_time($1)
-          end
-        end
-
-        property_supported :updated_on do
-          if content_for_scanner =~ /Updated Date: (.+?)\n/
-            parse_time($1)
-          end
-        end
-
-        property_supported :expires_on do
-          if content_for_scanner =~ /Registry Expiry Date: (.+?)\n/
-            parse_time($1)
-          end
-        end
-
-        property_supported :registrar do
-          if content_for_scanner =~ /Sponsoring Registrar: (.+?)\n/
-            Record::Registrar.new(
-                :name         => $1,
-                :organization => nil,
-                :url          => content_for_scanner.slice(/Sponsoring Registrar URL: (.+)\n/, 1)
-            )
-          end
-        end
-
-        property_supported :nameservers do
-          content_for_scanner.scan(/Name Server: (.+)\n/).flatten.map do |name|
-            Record::Nameserver.new(:name => name)
-          end
-        end
-
-        def statuses
-          content_for_scanner.scan(/Domain Status: (.+)\n/).flatten.map(&:downcase)
-        end
-
-		# The following methods are implemented by Yang Li on 01/24/2013
-		# ----------------------------------------------------------------------------
-
-		property_supported :domain_id do
-          return $1 if content_for_scanner =~ /Domain ID:\s+(.*)\n/i
-        end
-		
-        property_supported :registrant_contacts do
-          build_contact("Registrant", Whois::Record::Contact::TYPE_REGISTRANT)
-        end
-
-        property_supported :admin_contacts do
-          build_contact("Admin", Whois::Record::Contact::TYPE_ADMIN)
-        end
-
-        property_supported :technical_contacts do
-          build_contact("Tech", Whois::Record::Contact::TYPE_TECHNICAL)
-        end
-
-        property_supported :billing_contacts do
-          build_contact("Billing", Whois::Record::Contact::TYPE_BILLING)
-        end
-		
-      private
-
-        def parse_time(value)
-          # Hack to remove usec. Do you know a better way?
-          Time.utc(*Time.parse(value).to_a)
-        end
-
-        def build_contact(element, type)
-          reg=Record::Contact.new(:type => type)
-		  strs=''
-		  content_cleanup=content_for_scanner.split(%r{/n}).delete_if { |x| x !~ /(.+):(.+)/ }
-		  content_cleanup.join('\n').scan(/^#{element}(.+):(.+)\n/).map do |entry|
-              reg["id"]=entry[1].strip if entry[0] =~ /ID/i
-              reg["name"]=entry[1].strip if entry[0] =~ /Name/i
-              reg["organization"]=entry[1].strip if entry[0]=~ /Organization/i
-              if (entry[0]=~ /Street/i) 
-				strs=strs+entry[1].strip+", "
-			  end
-			  reg["address"]=strs
-              reg["city"]= entry[1].strip if entry[0]=~ /City/i
-              reg["zip"]=entry[1].strip if entry[0]=~ /Postal\sCode/i
-              reg["state"]=entry[1].strip if entry[0]=~ /State\/Province/i
-              reg["country_code"]=entry[1].strip if entry[0]=~ /Country/i
-			  reg["email"]=entry[1].strip if entry[0]=~ /Email/i
-			  reg["phone"]=entry[1].strip if entry[0] =~ /Phone$/i 
-			  reg["fax"]=entry[1].strip if (entry[0] =~ /Fax$/i) 
-          end
-		  return reg
-        end	
-		# ----------------------------------------------------------------------------
-		
-      end
-    end
-  end
-end