wmap 2.7.6 → 2.8.2

data/test/cidr_tracker_test.rb

@@ -0,0 +1,36 @@
+#--
+# Wmap
+#
+# A pure Ruby library for the Internet web application discovery and tracking.
+#
+# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
+#++
+# Unit Test File for Wmap::DomainTracker.instance class
+
+require "minitest/autorun"
+require "Wmap"
+
+class CidrTrackerTest < MiniTest::Unit::TestCase
+	include Wmap::Utils
+
+	def test_cidr_add
+		w = Wmap::CidrTracker.new
+		w.add("192.168.1.0/24")
+		assert_equal true, w.known_cidr_blks.key?("192.168.1.0/24")
+	end
+
+	def test_cidr_delete
+		w = Wmap::CidrTracker.new
+		w.add("10.0.0.0/8")
+		w.delete("10.0.0.0/8")
+		assert_equal false, w.known_cidr_blks.key?("10.0.0.0/8")
+	end
+
+	def test_ip_trusted?
+		w = Wmap::CidrTracker.new
+		w.add("192.168.1.0/24")
+		assert_equal true, w.ip_trusted?("192.168.1.1")
+		assert_equal true, w.ip_trusted?("192.168.1.255")
+	end
+
+end

data/test/utils_test.rb

@@ -12,7 +12,7 @@ require "Wmap"
 
 class UtilsTest < MiniTest::Unit::TestCase
 	include Wmap::Utils
-	
+
 	def test_sld_domain_conversion
 		assert_equal "yahoo.com", get_domain_root("yahoo.com")
 	end
@@ -28,75 +28,75 @@ class UtilsTest < MiniTest::Unit::TestCase
 	def test_is_domain_root_case_1?
 		assert_equal false, is_domain_root?("www.yahoo.co.uk")
 	end
-	
+
 	def test_is_domain_root_case_2?
 		assert_equal true, is_domain_root?("yahoo.co.uk")
 	end
-	
+
 	def test_get_sub_domain
 		assert_equal "mail.yahoo.co.uk", get_sub_domain("www.mail.yahoo.co.uk")
 	end
 
 	def test_is_url_case_1?
 		assert_equal true, is_url?("http://www.mail.yahoo.co.uk/")
-	end	
+	end
 
 	def test_is_url_case_2?
 		assert_equal true, is_url?("https://www.mail.yahoo.co.uk/")
-	end		
+	end
 
 	def test_is_url_case_3?
 		assert_equal false, is_url?("http://www.mail.yahoo.uii/")
-	end	
+	end
 
 	def test_is_url_case_4?
 		assert_equal false, is_url?("http:\\www.mail.yahoo.co.uk")
-	end	
-	
+	end
+
 	def test_is_ssl?
 		assert_equal false, is_ssl?("http://www.mail.yahoo.co.uk/")
-	end	
-	
+	end
+
 	def test_is_site?
 		assert_equal false, is_site?("https://login.yahoo.com/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
-	end	
-	
+	end
+
 	def test_url_2_host
 		assert_equal "login.yahoo.com", url_2_host("https://login.yahoo.com/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
-	end	
-	
+	end
+
 	def test_url_2_site_case_1
 		assert_equal "https://login.yahoo.com/", url_2_site("https://login.yahoo.com/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
-	end	
+	end
 
 	def test_url_2_site_case_2
 		assert_equal "https://login.yahoo.com/", url_2_site("https://login.yahoo.com?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
-	end	
+	end
 
 	def test_url_2_site_case_3
 		assert_equal "https://login.yahoo.com/", url_2_site("https://login.yahoo.com#.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
-	end	
-	
+	end
+
 	def test_url_2_path
 		assert_equal "/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com", url_2_path("https://login.yahoo.com/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
-	end	
+	end
 
 	def test_urls_on_same_domain?
 		assert_equal true, urls_on_same_domain?("https://login.yahoo.com/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com", "https://us-mg4.mail.yahoo.com/neo/launch?.rand=8hjd08hc6t1lq")
-	end	
+	end
 
 	def test_host_2_url_case_1
 		assert_equal "https://mail.yahoo.com/", host_2_url("mail.yahoo.com",443)
-	end	
+	end
 
 	def test_host_2_url_case_2
 		assert_equal "http://mail.yahoo.com/", host_2_url("mail.yahoo.com")
-	end	
-	
+	end
+
 	def test_make_absolute
 		assert_equal "http://games.yahoo.com/game/the-magic-snowman-flash.html", make_absolute("http://games.yahoo.com/","game/the-magic-snowman-flash.html")
 	end
-	
+
 	def test_create_absolute_url_from_base
 		assert_equal "http://images.search.yahoo.com/search/images?p=raiders", create_absolute_url_from_base("http://images.search.yahoo.com/images","/search/images?p=raiders")
 	end
@@ -108,7 +108,7 @@ class UtilsTest < MiniTest::Unit::TestCase
 	def test_normalize_url_case_1
 		assert_equal "http://images.search.yahoo.com/images/search/images?p=raiders", normalize_url("http://images.search.yahoo.com/./images/search/images?p=raiders")
 	end
-	
+
 	def test_normalize_url_case_2
 		assert_equal "http://images.search.yahoo.com/images/search/images?p=raiders", normalize_url("http://images.search.yahoo.com/../images/../search/images?p=raiders")
 	end
@@ -116,53 +116,58 @@ class UtilsTest < MiniTest::Unit::TestCase
 	def test_normalize_url_case_3
 		assert_equal "http://images.search.yahoo.com/images/search/images?p=raiders", normalize_url("http://images.search.yahoo.com./../images/../search/images?p=raiders")
 	end
-	
+
 	def test_is_ip_case_1?
 		assert_equal false, is_ip?("256.2.3.1")
-	end	
+	end
 
 	def test_is_ip_case_2?
 		assert_equal false, is_ip?("25.2.3.1.22")
-	end	
+	end
 
 	def test_is_ip_case_3?
 		assert_equal true, is_ip?("196.168.230.1")
-	end	
+	end
 
 	def test_is_fqdn_case_1?
 		assert_equal true, is_fqdn?("images.search.yahoo.com")
-	end	
+	end
 
 	def test_is_fqdn_case_2?
 		assert_equal true, is_fqdn?("yahoo.com")
-	end		
-	
+	end
+
 	def test_is_fqdn_case_3?
-		assert_equal false, is_fqdn?("images.search.yahoo")
-	end	
-	
+		# according to latest tlds list - http://data.iana.org/TLD/tlds-alpha-by-domain.txt
+		assert_equal true, is_fqdn?("images.search.yahoo")
+	end
+
 	def test_is_fqdn_case_4?
 		assert_equal false, is_fqdn?("images")
-	end	
-	
+	end
+
+	def test_is_fqdn_case_5?
+		assert_equal false, is_fqdn?("images.search.gargle")
+	end
+
 	def test_is_cidr_case_1?
 		assert_equal false, is_cidr?("196.168.230.1")
-	end	
+	end
 
 	def test_is_cidr_case_2?
 		assert_equal false, is_cidr?("196.168.2.257/12")
-	end	
-	
+	end
+
 	def test_is_cidr_case_3?
 		assert_equal true, is_cidr?("196.168.2.25/12")
-	end	
-	
+	end
+
 	def test_cidr_2_ips
 		assert_equal ["192.168.1.1"], cidr_2_ips("192.168.1.1/32")
-	end	
-	
+	end
+
 	def test_sort_ips
 		assert_equal ["192.168.1.1", "192.168.1.2", "192.168.2.1"], sort_ips(["192.168.1.2", "192.168.2.1","192.168.1.1"])
-	end		
-	
+	end
+
 end

data/version.txt

@@ -3,8 +3,8 @@
 ###############################################################################
 package = wmap
 # wmap version 2.0 == web_discovery version 1.5.3
-version = 2.7.6
-date = 2020-03-20
+version = 2.8.2
+date = 2021-07-23
 
 author = Sam (Yang) Li
 email = yang.li@owasp.org

data/wmap.gemspec

@@ -36,7 +36,7 @@ Gem::Specification.new do |s|
   s.description = "wmap is written to perform Internet web application / service discovery. The discovery results are designed to be automatically tracked by the software."
   s.email = info["email"]
   s.executables = ["wmap","wscan","wadd","wadds","wdel","wcheck","wdump","spiderBot","googleBot","updateAll","prime","deprime","refresh","trust","trusts","distrust","run_tests"]
-  s.files = ["CHANGELOG.md", "TODO", "settings/discovery_ports","data/","LICENSE.txt",
+  s.files = ["CHANGELOG.md", "TODO", "settings/discovery_ports", "LICENSE.txt",
 							"version.txt","README.md", "wmap.gemspec"]
   s.files += Dir['lib/*.rb'] + Dir['lib/wmap/*.rb'] + Dir['lib/wmap/**/*'] + Dir['bin/*'] + Dir['settings/*'] + Dir['demos/*'] + Dir['test/*'] + Dir['dicts/*']
   #s.homepage = "none"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wmap
 version: !ruby/object:Gem::Version
-  version: 2.7.6
+  version: 2.8.2
 platform: ruby
 authors:
 - Sam (Yang) Li
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-03-20 00:00:00.000000000 Z
+date: 2021-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dnsruby
@@ -234,7 +234,6 @@ files:
 - LICENSE.txt
 - README.md
 - TODO
-- bin/RHPG
 - bin/deprime
 - bin/distrust
 - bin/googleBot
@@ -251,7 +250,6 @@ files:
 - bin/wdel
 - bin/wdump
 - bin/wmap
-- bin/wmaps
 - bin/wscan
 - demos/bruter.rb
 - demos/dns_brutes.rb
@@ -303,11 +301,12 @@ files:
 - lib/wmap/utils/logger.rb
 - lib/wmap/utils/url_magic.rb
 - lib/wmap/utils/utils.rb
+- lib/wmap/utils/wp_detect.rb
 - lib/wmap/whois.rb
 - settings/discovery_ports
 - settings/google_keywords.txt
 - settings/google_locator.txt
-- settings/tag_signatures
+- test/cidr_tracker_test.rb
 - test/domain_tracker_test.rb
 - test/utils_test.rb
 - version.txt
@@ -334,9 +333,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: wmap
-rubygems_version: 2.7.10
-signing_key: 
+rubygems_version: 3.0.9
+signing_key:
 specification_version: 4
 summary: A pure Ruby web application and service discovery API.
 test_files: []

data/bin/RHPG

@@ -1,107 +0,0 @@
-#!/usr/bin/env ruby
-# Executable to lookup then merge site tech details into the RHPG asset spreadsheet in CSV format only
-#
-## Usage: RHPG [RHPG.csv]
-require "wmap"
-require "csv"
-include Wmap::Utils
-
-def print_usage
-	puts "Program to lookup then merge the site details into RHPG asset spreadsheet. \nUsage: RHPG [RHPG.csv]"
-end
-
-# Lookup the site store for a domain; then return the fingger print info of the site
-def site_tracker_lookup(domain)
-  tracker=Wmap::SiteTracker.instance
-	tracker.verbose=false
-  #first order search
-  tracker.known_sites.each do |key,val|
-    if key.include?(domain.strip.downcase) && key.include?("https")
-      tracker=nil
-      return [key] + val.values
-    end
-  end
-  #second order search
-  tracker.known_sites.each do |key,val|
-    if key.include?(domain.strip.downcase)
-      tracker=nil
-      return [key] + val.values
-    end
-  end
-  tracker=nil
-  return [nil]*9
-end
-
-# look up the wp site data store for a domain; then return the wp finger print info: [is_wp?,wp_ver]
-def wp_tracker_lookup(domain)
-  tracker=Wmap::WpTracker.new(:verbose=>false)
-	# first order
-	tracker.known_wp_sites.each do |key,val|
-		if key.include?(domain.strip.downcase) && val
-			ver=tracker.wp_ver(key)
-			tracker=nil
-			return [val,ver]
-		end
-	end
-  # second order
-  tracker.known_wp_sites.each do |key,val|
-    if key.include?(domain.strip.downcase) && key.include?("https") && val
-      tracker=nil
-      return [val,nil]
-    end
-  end
-  # third order
-  tracker.known_wp_sites.each do |key,val|
-    if key.include?(domain.strip.downcase)
-      tracker=nil
-      return [val,nil]
-    end
-  end
-  tracker=nil
-  return [nil,nil]
-end
-
-# perform the wpscan on a site
-def wpscan(domain)
-	url=site_tracker_lookup(domain)[0]
-	return nil if url.nil?
-	if url.include?("https")
-		command="wpscan --disable-tls-checks --ignore-main-redirect --url=" + url + " -o " + domain + ".wpscan"
-	else
-		command="wpscan --ignore-main-redirect --url=" + url + " -o " + domain + ".wpscan"
-	end
-	system(command)
-end
-
-puts Wmap.banner
-print_usage
-
-# open output file to write
-CSV.open("output.csv", "wb") do |csv|
-  cnt=1
-  # open RHPG input file to read
-  CSV.foreach(ARGV[0]) do |row|
-    puts "Processing row #{cnt}"
-    #puts row.inspect
-    my_row=Array.new
-    if cnt > 1
-      if is_domain?(row[0])
-=begin
-				if row[3] =~ /Keep/i && row[3] != /Redirect/i
-					unless File.exist?(row[0]+".wpscan")
-						wpscan(row[0])
-					end
-				end
-=end
-        my_row = row + site_tracker_lookup(row[0]) + wp_tracker_lookup(row[0])
-      else
-        my_row = row + [nil]*10
-      end
-    else
-      my_row = row + ["Website","Primary IP","Port","Hosting Status","Server","Response Code","MD5 Finger-print","Redirection","Timestamp", "WordPress", "WordPress Version"]
-    end
-    cnt+=1
-    csv << my_row
-  end
-  puts "All done. "
-end

data/bin/wmaps

@@ -1,23 +0,0 @@
-#!/usr/bin/env ruby
-# script to automate the new site discovery through by crawling all unique sites in the site store
-require "wmap"
-require "parallel"
-
-def wmap_worker(domain)
-   cmd = "wmap " + domain
-   puts "wmap discovery on domain: ", domain
-   system(cmd)
-end
-
-
-tracker=Wmap::DomainTracker.instance
-Parallel.map(tracker.known_internet_domains.keys, :in_processes => 10) { |target|
-  puts "Working on #{target} ..." if @verbose
-  wmap_worker(target)
-}
-=begin
-tracker.known_internet_domains.keys.map do |domain|
-  wmap_worker(domain)
-end
-=end
-tracker=nil

data/settings/tag_signatures

@@ -1,6 +0,0 @@
-# Adware signature file: signture string, description
-gtag.js, Google / DoubleClick Floodlight Tag
-analytics.js, Google Universal Analytics Tag
-ga.js, Google Analytics Tag
-utag.js, Tealium Tag
-all.js, Facebook Tag