wmap 2.4.4

data/test/domain_tracker_test.rb

@@ -0,0 +1,31 @@
+#--
+# Wmap
+#
+# A pure Ruby library for the Internet web application discovery and tracking.
+#
+# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
+#++
+# Unit Test File for Wmap::DomainTracker class
+
+require "minitest/autorun"
+require "Wmap"
+
+class DomainTrackerTest < MiniTest::Unit::TestCase
+	include Wmap::Utils
+
+	def test_domain_known_case_1?
+		assert_equal false, Wmap::DomainTracker.new.domain_known?("yahoo.com")
+	end
+
+	def test_domain_known_case_2?
+		assert_equal true, Wmap::DomainTracker.new.domain_known?("YourDomain.co.uk")
+	end
+
+	def test_domain_known_case_3?
+		assert_equal false, Wmap::DomainTracker::SubDomain.new.domain_known?("mail.yahoo.com")
+	end
+
+	def test_domain_known_case_4?
+		assert_equal true, Wmap::DomainTracker::SubDomain.new.domain_known?("YourHost.YourDomain.co.uk")
+	end
+end

data/test/utils_test.rb

@@ -0,0 +1,168 @@
+#--
+# Wmap
+#
+# A pure Ruby library for the Internet web application discovery and tracking.
+#
+# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
+#++
+# Unit Test File for Wmap::Utils module
+
+require "minitest/autorun"
+require "Wmap"
+
+class UtilsTest < MiniTest::Unit::TestCase
+	include Wmap::Utils
+	
+	def test_sld_domain_conversion
+		assert_equal "yahoo.com", get_domain_root("yahoo.com")
+	end
+
+	def test_host_2_sld_domain_coversion
+		assert_equal "yahoo.com", get_domain_root("www.yahoo.com")
+	end
+
+	def test_host_2_cclld_domain_coversion
+		assert_equal "yahoo.co.uk", get_domain_root("www.yahoo.co.uk")
+	end
+
+	def test_is_domain_root_case_1?
+		assert_equal false, is_domain_root?("www.yahoo.co.uk")
+	end
+	
+	def test_is_domain_root_case_2?
+		assert_equal true, is_domain_root?("yahoo.co.uk")
+	end
+	
+	def test_get_sub_domain
+		assert_equal "mail.yahoo.co.uk", get_sub_domain("www.mail.yahoo.co.uk")
+	end
+
+	def test_is_url_case_1?
+		assert_equal true, is_url?("http://www.mail.yahoo.co.uk/")
+	end	
+
+	def test_is_url_case_2?
+		assert_equal true, is_url?("https://www.mail.yahoo.co.uk/")
+	end		
+
+	def test_is_url_case_3?
+		assert_equal false, is_url?("http://www.mail.yahoo.uii/")
+	end	
+
+	def test_is_url_case_4?
+		assert_equal false, is_url?("http:\\www.mail.yahoo.co.uk")
+	end	
+	
+	def test_is_ssl?
+		assert_equal false, is_ssl?("http://www.mail.yahoo.co.uk/")
+	end	
+	
+	def test_is_site?
+		assert_equal false, is_site?("https://login.yahoo.com/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
+	end	
+	
+	def test_url_2_host
+		assert_equal "login.yahoo.com", url_2_host("https://login.yahoo.com/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
+	end	
+	
+	def test_url_2_site_case_1
+		assert_equal "https://login.yahoo.com/", url_2_site("https://login.yahoo.com/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
+	end	
+
+	def test_url_2_site_case_2
+		assert_equal "https://login.yahoo.com/", url_2_site("https://login.yahoo.com?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
+	end	
+
+	def test_url_2_site_case_3
+		assert_equal "https://login.yahoo.com/", url_2_site("https://login.yahoo.com#.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
+	end	
+	
+	def test_url_2_path
+		assert_equal "/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com", url_2_path("https://login.yahoo.com/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
+	end	
+
+	def test_urls_on_same_domain?
+		assert_equal true, urls_on_same_domain?("https://login.yahoo.com/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com", "https://us-mg4.mail.yahoo.com/neo/launch?.rand=8hjd08hc6t1lq")
+	end	
+
+	def test_host_2_url_case_1
+		assert_equal "https://mail.yahoo.com/", host_2_url("mail.yahoo.com",443)
+	end	
+
+	def test_host_2_url_case_2
+		assert_equal "http://mail.yahoo.com/", host_2_url("mail.yahoo.com")
+	end	
+	
+	def test_make_absolute
+		assert_equal "http://games.yahoo.com/game/the-magic-snowman-flash.html", make_absolute("http://games.yahoo.com/","game/the-magic-snowman-flash.html")
+	end
+	
+	def test_create_absolute_url_from_base
+		assert_equal "http://images.search.yahoo.com/search/images?p=raiders", create_absolute_url_from_base("http://images.search.yahoo.com/images","/search/images?p=raiders")
+	end
+
+	def test_create_absolute_url_from_context
+		assert_equal "http://images.search.yahoo.com/images/search/images?p=raiders", create_absolute_url_from_context("http://images.search.yahoo.com/images/logo.png","/search/images?p=raiders")
+	end
+
+	def test_normalize_url_case_1
+		assert_equal "http://images.search.yahoo.com/images/search/images?p=raiders", normalize_url("http://images.search.yahoo.com/./images/search/images?p=raiders")
+	end
+	
+	def test_normalize_url_case_2
+		assert_equal "http://images.search.yahoo.com/images/search/images?p=raiders", normalize_url("http://images.search.yahoo.com/../images/../search/images?p=raiders")
+	end
+
+	def test_normalize_url_case_3
+		assert_equal "http://images.search.yahoo.com/images/search/images?p=raiders", normalize_url("http://images.search.yahoo.com./../images/../search/images?p=raiders")
+	end
+	
+	def test_is_ip_case_1?
+		assert_equal false, is_ip?("256.2.3.1")
+	end	
+
+	def test_is_ip_case_2?
+		assert_equal false, is_ip?("25.2.3.1.22")
+	end	
+
+	def test_is_ip_case_3?
+		assert_equal true, is_ip?("196.168.230.1")
+	end	
+
+	def test_is_fqdn_case_1?
+		assert_equal true, is_fqdn?("images.search.yahoo.com")
+	end	
+
+	def test_is_fqdn_case_2?
+		assert_equal true, is_fqdn?("yahoo.com")
+	end		
+	
+	def test_is_fqdn_case_3?
+		assert_equal false, is_fqdn?("images.search.yahoo")
+	end	
+	
+	def test_is_fqdn_case_4?
+		assert_equal false, is_fqdn?("images")
+	end	
+	
+	def test_is_cidr_case_1?
+		assert_equal false, is_cidr?("196.168.230.1")
+	end	
+
+	def test_is_cidr_case_2?
+		assert_equal false, is_cidr?("196.168.2.257/12")
+	end	
+	
+	def test_is_cidr_case_3?
+		assert_equal true, is_cidr?("196.168.2.25/12")
+	end	
+	
+	def test_cidr_2_ips
+		assert_equal ["192.168.1.1"], cidr_2_ips("192.168.1.1/32")
+	end	
+	
+	def test_sort_ips
+		assert_equal ["192.168.1.1", "192.168.1.2", "192.168.2.1"], sort_ips(["192.168.1.2", "192.168.2.1","192.168.1.1"])
+	end		
+	
+end

data/version.txt

@@ -0,0 +1,13 @@
+###############################################################################
+# Simple Program Version Information
+###############################################################################
+package = wmap
+# wmap version 2.0 == web_discovery version 1.5.3
+version = 2.4.4
+date = 2018-01-16
+
+author = Yang Li
+email = yang.li@owasp.org
+linkedin = http://www.linkedin.com/in/yangli8/
+github = https://github.com/yangsec888/wmap 
+###############################################################################

data/wmap.gemspec

@@ -0,0 +1,49 @@
+#--
+# Wmap
+#
+# A pure Ruby library for Internet web application discovery and tracking.
+#
+# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
+#++
+
+# -*- encoding: utf-8 -*-
+VERSION = File.dirname(__FILE__) + "/version.txt"
+
+# Simple parser for the project version file
+info=Hash.new
+f=File.open(VERSION,'r')
+f.each do |line|
+	line.chomp!
+	case line
+	when /^(\s)*#/
+		next
+	when /\=/ 
+		entry=line.split("=").map! {|x| x.strip}
+		info[entry[0]]=entry[1]
+	end
+end
+f.close
+#
+
+Gem::Specification.new do |s|
+  s.name = info["package"]
+  s.version = info["version"]
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = info["author"]
+  s.homepage = info["github"]
+  s.licenses = ["MIT"]
+  s.date = info["date"]
+  s.description = "wmap is written to perform Internet web application / service discovery. The discovery results are designed to be automatically tracked by the software."
+  s.email = info["email"]
+  s.executables = ["wmap","wscan","wadd","wadds","wdel","wcheck","wdump","spiderBot","googleBot","updateAll","prime","deprime","refresh","trust","distrust","run_tests"]
+  s.files = ["CHANGELOG.md", "TODO", "settings/discovery_ports","settings/google_keywords.txt","settings/google_locator.txt","data/cidrs","data/domains","data/sub_domains","data/hosts","data/sites","data/deactivated_sites","data/prime_hosts","lib/wmap/cidr_tracker.rb","lib/wmap/dns_bruter.rb","lib/wmap/domain_tracker.rb","lib/wmap/domain_tracker/sub_domain.rb","lib/wmap/host_tracker.rb","lib/wmap/host_tracker/primary_host.rb","lib/wmap/network_profiler.rb","lib/wmap/port_scanner.rb","lib/wmap/site_tracker.rb","lib/wmap/site_tracker/deactivated_site.rb","lib/wmap/url_checker.rb","lib/wmap/url_crawler.rb","lib/wmap/geoip_tracker.rb","lib/wmap/google_search_scraper.rb","lib/wmap/utils/logger.rb","lib/wmap/utils/domain_root.rb","lib/wmap/utils/url_magic.rb","lib/wmap/utils/utils.rb","lib/wmap/whois.rb","lib/wmap.rb","LICENSE.txt","version.txt","README.rdoc",  "wmap.gemspec"]
+  s.files += Dir['bin/*'] + Dir['demos/*'] + Dir['test/*'] + Dir['ruby_whois_patches/*'] + Dir['dicts/*'] + Dir['logs/wmap.log']
+  #s.homepage = "none"
+  s.post_install_message = "*"*80 + "\n\nThank you for installing the wmap gem - a pure Ruby library for Internet web application discovery and tracking. Please refer to the README.rdoc for more information of using this gem.  \n\n" + "*"*80 + "\n"
+  s.require_paths = ["lib"]
+  s.required_ruby_version = Gem::Requirement.new(">= 1.9.2")
+  s.rubyforge_project = "wmap"
+  s.rubygems_version = "1.8.24"
+  s.summary = "A pure Ruby web application and service discovery API."
+
+end

metadata

@@ -0,0 +1,202 @@
+--- !ruby/object:Gem::Specification
+name: wmap
+version: !ruby/object:Gem::Version
+  version: 2.4.4
+platform: ruby
+authors:
+- Yang Li
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-01-16 00:00:00.000000000 Z
+dependencies: []
+description: wmap is written to perform Internet web application / service discovery.
+  The discovery results are designed to be automatically tracked by the software.
+email: yang.li@owasp.org
+executables:
+- wmap
+- wscan
+- wadd
+- wadds
+- wdel
+- wcheck
+- wdump
+- spiderBot
+- googleBot
+- updateAll
+- prime
+- deprime
+- refresh
+- trust
+- distrust
+- run_tests
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.rdoc
+- TODO
+- bin/deprime
+- bin/distrust
+- bin/googleBot
+- bin/prime
+- bin/refresh
+- bin/run_tests
+- bin/spiderBot
+- bin/trust
+- bin/updateAll
+- bin/wadd
+- bin/wadds
+- bin/wcheck
+- bin/wdel
+- bin/wdump
+- bin/wmap
+- bin/wscan
+- data/cidrs
+- data/deactivated_sites
+- data/domains
+- data/hosts
+- data/prime_hosts
+- data/sites
+- data/sub_domains
+- demos/bruter.rb
+- demos/dns_brutes.rb
+- demos/filter_cidr.rb
+- demos/filter_crawls.rb
+- demos/filter_domain.rb
+- demos/filter_geoip.rb
+- demos/filter_known_services.rb
+- demos/filter_netinfo.rb
+- demos/filter_prime.rb
+- demos/filter_profiler.rb
+- demos/filter_redirection.rb
+- demos/filter_site.rb
+- demos/filter_siteip.rb
+- demos/filter_status.rb
+- demos/filter_timestamp.rb
+- demos/filter_url.rb
+- demos/new_fnd.rb
+- demos/nmap_parser.pl
+- demos/site_format.rb
+- demos/whois_domain.rb
+- dicts/GeoIP.dat
+- dicts/GeoIPASNum.dat
+- dicts/GeoLiteCity.dat
+- dicts/ccsld.txt
+- dicts/cctld.txt
+- dicts/gtld.txt
+- dicts/hostnames-dict.big
+- dicts/hostnames-dict.txt
+- lib/wmap.rb
+- lib/wmap/cidr_tracker.rb
+- lib/wmap/dns_bruter.rb
+- lib/wmap/domain_tracker.rb
+- lib/wmap/domain_tracker/sub_domain.rb
+- lib/wmap/geoip_tracker.rb
+- lib/wmap/google_search_scraper.rb
+- lib/wmap/host_tracker.rb
+- lib/wmap/host_tracker/primary_host.rb
+- lib/wmap/network_profiler.rb
+- lib/wmap/port_scanner.rb
+- lib/wmap/site_tracker.rb
+- lib/wmap/site_tracker/deactivated_site.rb
+- lib/wmap/url_checker.rb
+- lib/wmap/url_crawler.rb
+- lib/wmap/utils/domain_root.rb
+- lib/wmap/utils/logger.rb
+- lib/wmap/utils/url_magic.rb
+- lib/wmap/utils/utils.rb
+- lib/wmap/whois.rb
+- logs/wmap.log
+- ruby_whois_patches/base_cocca2.rb
+- ruby_whois_patches/kero.yachay.pe.rb
+- ruby_whois_patches/whois.PublicDomainRegistry.com.rb
+- ruby_whois_patches/whois.above.com.rb
+- ruby_whois_patches/whois.adamsnames.tc.rb
+- ruby_whois_patches/whois.aeda.net.ae.rb
+- ruby_whois_patches/whois.ai.rb
+- ruby_whois_patches/whois.arnes.si.rb
+- ruby_whois_patches/whois.ascio.com.rb
+- ruby_whois_patches/whois.cnnic.cn.rb
+- ruby_whois_patches/whois.corporatedomains.com.rb
+- ruby_whois_patches/whois.crsnic.net.rb
+- ruby_whois_patches/whois.denic.de.rb
+- ruby_whois_patches/whois.dk-hostmaster.dk.rb
+- ruby_whois_patches/whois.dns.be.rb
+- ruby_whois_patches/whois.dns.lu.rb
+- ruby_whois_patches/whois.dns.pl.rb
+- ruby_whois_patches/whois.dns.pt.rb
+- ruby_whois_patches/whois.domain.kg.rb
+- ruby_whois_patches/whois.domainregistry.my.rb
+- ruby_whois_patches/whois.domreg.lt.rb
+- ruby_whois_patches/whois.dot.tk.rb
+- ruby_whois_patches/whois.hkirc.hk.rb
+- ruby_whois_patches/whois.isnic.is.rb
+- ruby_whois_patches/whois.je.rb
+- ruby_whois_patches/whois.jprs.jp.rb
+- ruby_whois_patches/whois.kenic.or.ke.rb
+- ruby_whois_patches/whois.markmonitor.com.rb
+- ruby_whois_patches/whois.melbourneit.com.rb
+- ruby_whois_patches/whois.nic.as.rb
+- ruby_whois_patches/whois.nic.at.rb
+- ruby_whois_patches/whois.nic.ch.rb
+- ruby_whois_patches/whois.nic.cl.rb
+- ruby_whois_patches/whois.nic.ec.rb
+- ruby_whois_patches/whois.nic.im.rb
+- ruby_whois_patches/whois.nic.it.rb
+- ruby_whois_patches/whois.nic.lv.rb
+- ruby_whois_patches/whois.nic.ly.rb
+- ruby_whois_patches/whois.nic.mu.rb
+- ruby_whois_patches/whois.nic.mx.rb
+- ruby_whois_patches/whois.nic.net.sa.rb
+- ruby_whois_patches/whois.nic.or.kr.rb
+- ruby_whois_patches/whois.nic.tel.rb
+- ruby_whois_patches/whois.nic.tr.rb
+- ruby_whois_patches/whois.nic.us.rb
+- ruby_whois_patches/whois.nic.ve.rb
+- ruby_whois_patches/whois.norid.no.rb
+- ruby_whois_patches/whois.pandi.or.id.rb
+- ruby_whois_patches/whois.psi-usa.info.rb
+- ruby_whois_patches/whois.registro.br.rb
+- ruby_whois_patches/whois.registrygate.com.rb
+- ruby_whois_patches/whois.rrpproxy.net.rb
+- ruby_whois_patches/whois.sgnic.sg.rb
+- ruby_whois_patches/whois.srs.net.nz.rb
+- ruby_whois_patches/whois.tucows.com.rb
+- ruby_whois_patches/whois.twnic.net.tw.rb
+- settings/discovery_ports
+- settings/google_keywords.txt
+- settings/google_locator.txt
+- test/domain_tracker_test.rb
+- test/utils_test.rb
+- version.txt
+- wmap.gemspec
+homepage: https://github.com/yangsec888/wmap
+licenses:
+- MIT
+metadata: {}
+post_install_message: "********************************************************************************\n\nThank
+  you for installing the wmap gem - a pure Ruby library for Internet web application
+  discovery and tracking. Please refer to the README.rdoc for more information of
+  using this gem.  \n\n********************************************************************************\n"
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: wmap
+rubygems_version: 2.4.5.2
+signing_key: 
+specification_version: 4
+summary: A pure Ruby web application and service discovery API.
+test_files: []