wmap 2.7.6 → 2.7.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1f2c900ee2ac330d411f3f6496868ea519dd322b236cd00be0b5f3d1d8d43db
-  data.tar.gz: 054a534426e0b9b99730ac6252a9b0e2424866df914d02f6f7f1bdb925588bc1
+  metadata.gz: 559cac84dd38902d968cc9e7327e77115ae3f946020caa21b7743ceb5777a96f
+  data.tar.gz: a4dcc0eafc10d7497c47c1f9955774b880ae5cdafa5ed0c6904ef5362087bd98
 SHA512:
-  metadata.gz: c591b085614ec050ff8637750dfff57c3238ccf42c2fce9cab985ccd0f3e4761ed085632a9db7f4af87be39e2921b33d330c890c1af87213a134bdd37f43553d
-  data.tar.gz: 414720066fee25c30b41beb9c7529f7bfffda513e01e1bf4445cfefcd9ecbf2c421aa64db931a4d489845d664705ededb2474c3240750ad12cd2423b21078987
+  metadata.gz: 0b430ed1da47cefd8cb8a7bedddd75ed2e7b1dafbfe94cdb2185ad2eb7e26d69a1429e79a9ece6b3cc68d6d964161f092e1143a00c0c64e241e930304d9e5a65
+  data.tar.gz: fe4d50b292849e51c202f47083c0837228d11934eb0d71c5a6262da24467480ce26994f5db91bdae5cf3b52dc0e6ef4fef698396575e820779c7420c731d0d87

data/lib/wmap.rb

@@ -8,6 +8,7 @@
 require 'wmap/utils/domain_root'
 require 'wmap/utils/url_magic'
 require 'wmap/utils/logger'
+require 'wmap/utils/wp_detect'
 require 'wmap/utils/utils'
 require 'wmap/cidr_tracker'
 require 'wmap/domain_tracker'

data/lib/wmap/site_tracker/wp_tracker.rb

@@ -7,16 +7,14 @@
 #++
 require "parallel"
 #require "singleton"
-require "open-uri"
-require "open_uri_redirections"
-require "nokogiri"
-require "css_parser"
+
 
 module Wmap
 class SiteTracker
 
 class WpTracker < Wmap::SiteTracker
 	include Wmap::Utils
+	include Wmap::Utils::WpDetect
 	#include Singleton
 
 	attr_accessor :http_timeout, :max_parallel, :verbose, :sites_wp, :data_dir
@@ -152,27 +150,6 @@ class WpTracker < Wmap::SiteTracker
 	end
 	alias_method :adds, :bulk_add
 
-  # logic to determin if it's a wordpress site
-  def is_wp?(url)
-		site=url_2_site(url)
-		if wp_readme?(site)
-			found=true
-		elsif wp_css?(site)
-			found=true
-		elsif wp_meta?(site)
-			found=true
-		elsif wp_login?(site)
-			found=true
-		elsif wp_rpc?(site)
-			found=true
-		else
-			found=false
-		end
-		return found
-	rescue => ee
-		puts "Exception on method #{__method__}: #{ee}: #{url}" if @verbose
-	end
-
 	# Refresh one site entry then update the instance variable (cache)
 	def refresh (target,use_cache=false)
 		return add(target,use_cache)
@@ -209,207 +186,6 @@ class WpTracker < Wmap::SiteTracker
 	#	return Hash.new
 	end
 
-  # Wordpress detection checkpoint - readme.html
-  def wp_readme?(url)
-		site = url_2_site(url)
-    readme_url=site + "readme.html"
-    k=Wmap::UrlChecker.new
-    if k.response_code(readme_url) == 200
-      k=nil
-      doc=open_page(readme_url)
-      title=doc.css('title')
-      if title.to_s =~ /wordpress/i
-        return true
-      else
-        return false
-      end
-    else
-      k=nil
-      return false
-    end
-	rescue => ee
-		puts "Exception on method #{__method__} for site #{url}: #{ee}" if @verbose
-		return false
-  end
-
-  # Wordpress detection checkpoint - install.css
-  def wp_css?(url)
-		site = url_2_site(url)
-    css_url = site + "wp-admin/css/install.css"
-    k=Wmap::UrlChecker.new
-    if k.response_code(css_url) == 200
-      k=nil
-      parser = CssParser::Parser.new
-      parser.load_uri!(css_url)
-      rule = parser.find_by_selector('#logo a')
-      if rule.length >0
-        if rule[0] =~ /wordpress/i
-          return true
-        end
-      end
-    else
-      k=nil
-      return false
-    end
-    return false
-	rescue => ee
-		puts "Exception on method #{__method__} for site #{url}: #{ee}" if @verbose
-		return false
-  end
-
-  # Wordpress detection checkpoint - meta generator
-  def wp_meta?(url)
-		site=url_2_site(url)
-    k=Wmap::UrlChecker.new
-    if k.response_code(site) == 200
-      k=nil
-      doc=open_page(site)
-      meta=doc.css('meta')
-      if meta.to_s =~ /wordpress/i
-        return true
-      else
-        return false
-      end
-    end
-		return false
-	rescue => ee
-		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
-		return false
-  end
-
-	# Wordpress detection checkpoint - wp-login
-  def wp_login?(url)
-		site=url_2_site(url)
-		login_url=site + "wp-login.php"
-    k=Wmap::UrlChecker.new
-    if k.response_code(login_url) == 200
-      k=nil
-      doc=open_page(login_url)
-      links=doc.css('link')
-      if links.to_s =~ /login.min.css/i
-        return true
-      else
-        return false
-      end
-    end
-		return false
-	rescue => ee
-		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
-		return false
-  end
-
-	# Wordpress detection checkpoint - xml-rpc
-  def wp_rpc?(url)
-		site=url_2_site(url)
-		rpc_url=site + "xmlrpc.php"
-    k=Wmap::UrlChecker.new
-		#puts "res code", k.response_code(rpc_url)
-    if k.response_code(rpc_url) == 405 # method not allowed
-      k=nil
-      return true
-    end
-		return false
-	rescue => ee
-		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
-		return false
-  end
-
-	# Extract the WordPress version
-	def wp_ver(url)
-		if !wp_ver_readme(url).nil?
-			puts "WordPress version found by wp_ver_readme method. " if @verbose
-			return wp_ver_readme(url)
-		elsif !wp_ver_login(url,"login.min.css").nil?
-			puts "WordPress version found by login.min.css file. " if @verbose
-			return wp_ver_login(url,"login.min.css")
-		elsif !wp_ver_login(url,"buttons.min.css").nil?
-			puts "WordPress version found by buttons.min.css file. " if @verbose
-			return wp_ver_login(url,"buttons.min.css")
-		elsif !wp_ver_login(url,"wp-admin.min.css").nil?
-			puts "WordPress version found by wp-admin.min.css file. " if @verbose
-			return wp_ver_login(url,"wp-admin.min.css")
-		elsif !wp_ver_meta(url).nil?
-			puts "WordPress version found by wp_ver_meta method. " if @verbose
-			return wp_ver_meta(url)
-		else
-			return nil
-		end
-	rescue => ee
-		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
-		return nil
-	end
-
-	# Identify wordpress version through the login page
-  def wp_ver_login(url,pattern)
-		puts "Check for #{pattern}" if @verbose
-		site=url_2_site(url)
-		login_url=site + "wp-login.php"
-    k=Wmap::UrlChecker.new
-		#puts "Res code: #{k.response_code(login_url)}" if @verbose
-    if k.response_code(login_url) == 200
-      doc=open_page(login_url)
-			#puts doc.inspect
-      links=doc.css('link')
-			#puts links.inspect if @verbose
-			links.each do |tag|
-	      if tag.to_s.include?(pattern)
-					puts tag.to_s if @verbose
-					k=nil
-	        return tag.to_s.scan(/[\d+\.]+\d+/).first
-	      end
-			end
-    end
-    k=nil
-    return nil
-	rescue => ee
-		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
-		return nil
-  end
-
-	# Identify wordpress version through the meta link
-  def wp_ver_meta(url)
-		site=url_2_site(url)
-    k=Wmap::UrlChecker.new
-    if k.response_code(site) == 200
-      doc=open_page(site)
-			#puts doc.inspect
-      meta=doc.css('meta')
-			#puts meta.inspect
-			meta.each do |tag|
-	      if tag['content'].to_s =~ /wordpress/i
-					#puts tag.to_s
-					k=nil
-	        return tag['content'].to_s.scan(/[\d+\.]+\d+/).first
-	      end
-			end
-    end
-    k=nil
-    return nil
-	rescue => ee
-		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
-		return nil
-  end
-
-	# Wordpress version detection via - readme.html
-  def wp_ver_readme(url)
-		site=url_2_site(url)
-    readme_url=site + "readme.html"
-    k=Wmap::UrlChecker.new
-		puts "Res code: #{k.response_code(readme_url)}" if @verbose
-    if k.response_code(readme_url) == 200
-      k=nil
-      doc=open_page(readme_url)
-			puts doc if @verbose
-      logo=doc.css('h1#logo')[0]
-      puts logo.inspect if @verbose
-			return logo.to_s.scan(/[\d+\.]+\d+/).first
-    end
-    k=nil
-    return nil
-	rescue => ee
-		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
-		return nil
-	end
 
 end
 end

data/lib/wmap/utils/url_magic.rb

@@ -263,7 +263,7 @@ module Wmap
 		return absolute_url
   rescue => ee
 		puts "Exception on method #{__method__}: #{ee}" if @verbose
-          return nil
+    return nil
   end
 
 	# Normalize the URL to a consistent manner in order to determine if a link has been visited or cached before
@@ -292,7 +292,6 @@ module Wmap
 		return url
 	end
 
-
 	# Test the URL and return the response code
 	def response_code (url)
 		puts "Check the http response code on the url: #{url}" if @verbose
@@ -344,6 +343,38 @@ module Wmap
 		return code
 	end
 
+	# Test the URL and return the response headers
+	def response_headers (url)
+		puts "Check the http response headers on the url: #{url}" if @verbose
+		raise "Invalid url: #{url}" unless is_url?(url)
+    headers = Hash.new
+		url=url.strip.downcase
+		timeo = Max_http_timeout/1000.0
+		uri = URI.parse(url)
+		http = Net::HTTP.new(uri.host, uri.port)
+		http.open_timeout = timeo
+		http.read_timeout = timeo
+		if (url =~ /https\:/i)
+			http.use_ssl = true
+			#http.ssl_version = :SSLv3
+			# Bypass the remote web server cert validation test
+			http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+		end
+		request = Net::HTTP::Get.new(uri.request_uri)
+		response = http.request(request)
+		puts "Server response the following: #{response}" if @verbose
+		response.each_header do |key,val|
+      puts "#{key} => #{val}" if @verbose
+      headers.merge!({key => val})
+    end
+		puts "Response headers on #{url}: #{headers}" if @verbose
+		return headers
+  rescue => ee
+		puts "Exception on method #{__method__}: #{ee}" if @verbose
+    return nil
+  end
+
+
   # Given an URL, open the page, then return the DOM text from a normal user perspective
   def open_page(url)
     args = {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE, allow_redirections: :safe, read_timeout: Max_http_timeout/1000}

data/lib/wmap/utils/wp_detect.rb

@@ -0,0 +1,354 @@
+#--
+# Wmap
+#
+# A pure Ruby library for Internet web application discovery and tracking.
+#
+# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
+#++
+
+# Utilities for wp_tracker class only; must use with other Utils modules.
+module Wmap
+ module Utils
+    module WpDetect
+	  extend self
+
+    # Main method to detect if it's a wordpress site
+    def is_wp?(url)
+  		site=url_2_site(url)
+  		if wp_readme?(site)
+  			return true
+  		elsif wp_css?(site)
+  			return true
+  		elsif wp_meta?(site)
+  			return true
+  		elsif wp_login?(site)
+  			return true
+  		elsif wp_rpc?(site)
+  			return true
+  		elsif wp_gen?(site)
+  			return true
+      elsif wp_load_styles?(site)
+  			return true
+  		else
+  			return false
+  		end
+  	rescue => ee
+  		puts "Exception on method #{__method__}: #{ee}: #{url}" if @verbose
+  	end
+
+    # Main method to extract the WordPress version
+  	def wp_ver(url)
+  		if !wp_ver_readme(url).nil?
+  			puts "WordPress version found by wp_ver_readme method. " if @verbose
+  			return wp_ver_readme(url)
+  		elsif !wp_ver_login(url,"login.min.css").nil?
+  			puts "WordPress version found by login.min.css file. " if @verbose
+  			return wp_ver_login(url,"login.min.css")
+  		elsif !wp_ver_login(url,"buttons.min.css").nil?
+  			puts "WordPress version found by buttons.min.css file. " if @verbose
+  			return wp_ver_login(url,"buttons.min.css")
+  		elsif !wp_ver_login(url,"wp-admin.min.css").nil?
+  			puts "WordPress version found by wp-admin.min.css file. " if @verbose
+  			return wp_ver_login(url,"wp-admin.min.css")
+  		elsif !wp_ver_meta(url).nil?
+  			puts "WordPress version found by wp_ver_meta method. " if @verbose
+  			return wp_ver_meta(url)
+  		elsif !wp_ver_generator(url).nil?
+  			puts "WordPress version found by wp_ver_generator method. " if @verbose
+  			return wp_ver_generator(url)
+      elsif !wp_ver_load_styles(url).nil?
+  			puts "WordPress version found by wp_ver_load_styles method. " if @verbose
+  			return wp_ver_load_styles(url)
+      else
+  			return nil
+  		end
+  	rescue => ee
+  		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
+  		return nil
+  	end
+
+    # Wordpress detection checkpoint - readme.html
+    def wp_readme?(url)
+  		site = url_2_site(url)
+      readme_url=site + "readme.html"
+      k=Wmap::UrlChecker.new
+      if k.response_code(readme_url) == 200
+        k=nil
+        doc=open_page(readme_url)
+        title=doc.css('title')
+        if title.to_s =~ /wordpress/i
+          return true
+        else
+          return false
+        end
+      else
+        k=nil
+        return false
+      end
+  	rescue => ee
+  		puts "Exception on method #{__method__} for site #{url}: #{ee}" if @verbose
+  		return false
+    end
+
+    # Wordpress detection checkpoint - install.css
+    def wp_css?(url)
+  		site = url_2_site(url)
+      css_url = site + "wp-admin/css/install.css"
+      k=Wmap::UrlChecker.new
+      if k.response_code(css_url) == 200
+        k=nil
+        parser = CssParser::Parser.new
+        parser.load_uri!(css_url)
+        rule = parser.find_by_selector('#logo a')
+        if rule.length >0
+          if rule[0] =~ /wordpress/i
+            return true
+          end
+        end
+      else
+        k=nil
+        return false
+      end
+      return false
+  	rescue => ee
+  		puts "Exception on method #{__method__} for site #{url}: #{ee}" if @verbose
+  		return false
+    end
+
+    # Wordpress detection checkpoint - WP meta tag
+    def wp_meta?(url)
+  		site=url_2_site(url)
+      k=Wmap::UrlChecker.new
+      if k.response_code(site) == 200
+        k=nil
+        doc=open_page(site)
+        meta=doc.css('meta')
+        if meta.to_s =~ /wordpress/i
+          return true
+        else
+          return false
+        end
+      end
+  		return false
+  	rescue => ee
+  		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
+  		return false
+    end
+
+  	# Wordpress detection checkpoint - WP generator tag
+    def wp_gen?(url)
+  		puts "#{__method__} check for #{url}" if @verbose
+  		site = url_2_site(url)
+  		gen_url_1 = site + "feed/"
+  		gen_url_2 = site + "comments/feed"
+      k=Wmap::UrlChecker.new
+      if k.response_code(gen_url_1) == 200
+        doc=open_page(gen_url_1)
+  		elsif k.response_code(gen_url_2) == 200
+  			doc=open_page(gen_url_2)
+  		else
+  			k=nil
+  			return false
+  		end
+  		#puts doc.inspect
+      gens=doc.css('generator')
+  		if gens.nil?
+  			k=nil
+  			return false
+  		end
+  		gens.each do |gen|
+  			if gen.text.to_s =~ /wordpress/i
+  				k=doc=nil
+          return true
+        end
+  		end
+  		k=doc=nil
+  		return false
+  	rescue => ee
+  		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
+  		return false
+    end
+
+  	# Wordpress detection checkpoint - wp-login
+    def wp_login?(url)
+  		site=url_2_site(url)
+  		login_url=site + "wp-login.php"
+      k=Wmap::UrlChecker.new
+      if k.response_code(login_url) == 200
+        k=nil
+        doc=open_page(login_url)
+        links=doc.css('link')
+        if links.to_s =~ /login.min.css/i
+          return true
+        else
+          return false
+        end
+      end
+  		return false
+  	rescue => ee
+  		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
+  		return false
+    end
+
+  	# Wordpress detection checkpoint - xml-rpc
+    def wp_rpc?(url)
+  		site=url_2_site(url)
+  		rpc_url=site + "xmlrpc.php"
+      k=Wmap::UrlChecker.new
+  		#puts "res code", k.response_code(rpc_url)
+      if k.response_code(rpc_url) == 405 # method not allowed
+        k=nil
+        return true
+      end
+  		return false
+  	rescue => ee
+  		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
+  		return false
+    end
+
+    # Wordpress detection checkpoint - /wp-admin/load-styles.php
+    def wp_load_styles?(url)
+  		site = url_2_site(url)
+      load_styles_url=site + "wp-admin/load-styles.php"
+      k=Wmap::UrlChecker.new
+      if k.response_code(load_styles_url) == 200 && k.response_headers(load_styles_url).keys.include?("etag")
+        k=nil
+        return true
+      else
+        k=nil
+        return false
+      end
+  	rescue => ee
+  		puts "Exception on method #{__method__} for site #{url}: #{ee}" if @verbose
+  		return false
+    end
+
+  	# Identify wordpress version through the login page
+    def wp_ver_login(url,pattern)
+  		puts "Check for #{pattern}" if @verbose
+  		site=url_2_site(url)
+  		login_url=site + "wp-login.php"
+      k=Wmap::UrlChecker.new
+  		#puts "Res code: #{k.response_code(login_url)}" if @verbose
+      if k.response_code(login_url) == 200
+        doc=open_page(login_url)
+  			#puts doc.inspect
+        links=doc.css('link')
+  			#puts links.inspect if @verbose
+  			links.each do |tag|
+  	      if tag.to_s.include?(pattern)
+  					puts tag.to_s if @verbose
+  					k=nil
+  	        return tag.to_s.scan(/[\d+\.]+\d+/).first
+  	      end
+  			end
+      end
+      k=nil
+      return nil
+  	rescue => ee
+  		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
+  		return nil
+    end
+
+  	# Identify wordpress version through the meta link
+    def wp_ver_meta(url)
+  		site=url_2_site(url)
+      k=Wmap::UrlChecker.new
+      if k.response_code(site) == 200
+        doc=open_page(site)
+  			#puts doc.inspect
+        meta=doc.css('meta')
+  			#puts meta.inspect
+  			meta.each do |tag|
+  	      if tag['content'].to_s =~ /wordpress/i
+  					#puts tag.to_s
+  					k=nil
+  	        return tag['content'].to_s.scan(/[\d+\.]+\d+/).first
+  	      end
+  			end
+      end
+      k=nil
+      return nil
+  	rescue => ee
+  		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
+  		return nil
+    end
+
+  	# Identify wordpress version through the generator tag: <generator>https://wordpress.org/?v=4.9.8</generator>
+    def wp_ver_generator(url)
+  		puts "#{__method__} check for #{url}" if @verbose
+  		site = url_2_site(url)
+  		gen_url_1 = site + "feed/"
+  		gen_url_2 = site + "comments/feed"
+      k=Wmap::UrlChecker.new
+      if k.response_code(gen_url_1) == 200
+        doc=open_page(gen_url_1)
+  		elsif k.response_code(gen_url_2) == 200
+  			doc=open_page(gen_url_2)
+  		else
+  			k=nil
+  			return nil
+  		end
+  		#puts doc.inspect
+      gens=doc.css('generator')
+  		if gens.nil?
+  			k=nil
+  			return nil
+  		end
+  		gens.each do |gen|
+  			if gen.text.to_s =~ /wordpress/i
+  				k=nil
+          return gen.text.to_s.scan(/[\d+\.]+\d+/).first
+        end
+  		end
+      k=doc=nil
+      return nil
+  	rescue => ee
+  		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
+  		return nil
+  	end
+
+  	# Wordpress version detection via - readme.html
+    def wp_ver_readme(url)
+  		site=url_2_site(url)
+      readme_url=site + "readme.html"
+      k=Wmap::UrlChecker.new
+  		puts "Res code: #{k.response_code(readme_url)}" if @verbose
+      if k.response_code(readme_url) == 200
+        k=nil
+        doc=open_page(readme_url)
+  			puts doc if @verbose
+        logo=doc.css('h1#logo')[0]
+        puts logo.inspect if @verbose
+  			return logo.to_s.scan(/[\d+\.]+\d+/).first
+      end
+      k=nil
+      return nil
+  	rescue => ee
+  		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
+  		return nil
+  	end
+
+    # Wordpress version detection via - /wp-admin/load-styles.php
+    def wp_ver_load_styles(url)
+  		site=url_2_site(url)
+      load_styles_url = site + "wp-admin/load-styles.php"
+      k=Wmap::UrlChecker.new
+      if k.response_code(load_styles_url) == 200
+        headers = k.response_headers(load_styles_url)
+        if headers.keys.include?("etag")
+          k=nil
+          return headers["etag"]
+        end
+      end
+      k=nil
+      return nil
+  	rescue => ee
+  		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
+  		return nil
+  	end
+
+
+  end
+end
+end

data/version.txt

@@ -3,8 +3,8 @@
 ###############################################################################
 package = wmap
 # wmap version 2.0 == web_discovery version 1.5.3
-version = 2.7.6
-date = 2020-03-20
+version = 2.7.7
+date = 2020-03-24
 
 author = Sam (Yang) Li
 email = yang.li@owasp.org

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wmap
 version: !ruby/object:Gem::Version
-  version: 2.7.6
+  version: 2.7.7
 platform: ruby
 authors:
 - Sam (Yang) Li
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-20 00:00:00.000000000 Z
+date: 2020-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dnsruby
@@ -303,6 +303,7 @@ files:
 - lib/wmap/utils/logger.rb
 - lib/wmap/utils/url_magic.rb
 - lib/wmap/utils/utils.rb
+- lib/wmap/utils/wp_detect.rb
 - lib/wmap/whois.rb
 - settings/discovery_ports
 - settings/google_keywords.txt