wmap 2.4.4 → 2.4.5

data/version.txt

@@ -3,11 +3,11 @@
 ###############################################################################
 package = wmap
 # wmap version 2.0 == web_discovery version 1.5.3
-version = 2.4.4
-date = 2018-01-16
+version = 2.4.5
+date = 2018-11-06
 
-author = Yang Li
+author = Sam (Yang) Li
 email = yang.li@owasp.org
 linkedin = http://www.linkedin.com/in/yangli8/
-github = https://github.com/yangsec888/wmap 
+github = https://github.com/yangsec888/wmap
 ###############################################################################

data/wmap.gemspec

@@ -3,7 +3,7 @@
 #
 # A pure Ruby library for Internet web application discovery and tracking.
 #
-# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
+# Copyright (c) 2012-2015 Sam (Yang) Li <yang.li@owasp.org>
 #++
 
 # -*- encoding: utf-8 -*-
@@ -17,7 +17,7 @@ f.each do |line|
 	case line
 	when /^(\s)*#/
 		next
-	when /\=/ 
+	when /\=/
 		entry=line.split("=").map! {|x| x.strip}
 		info[entry[0]]=entry[1]
 	end
@@ -36,12 +36,27 @@ Gem::Specification.new do |s|
   s.description = "wmap is written to perform Internet web application / service discovery. The discovery results are designed to be automatically tracked by the software."
   s.email = info["email"]
   s.executables = ["wmap","wscan","wadd","wadds","wdel","wcheck","wdump","spiderBot","googleBot","updateAll","prime","deprime","refresh","trust","distrust","run_tests"]
-  s.files = ["CHANGELOG.md", "TODO", "settings/discovery_ports","settings/google_keywords.txt","settings/google_locator.txt","data/cidrs","data/domains","data/sub_domains","data/hosts","data/sites","data/deactivated_sites","data/prime_hosts","lib/wmap/cidr_tracker.rb","lib/wmap/dns_bruter.rb","lib/wmap/domain_tracker.rb","lib/wmap/domain_tracker/sub_domain.rb","lib/wmap/host_tracker.rb","lib/wmap/host_tracker/primary_host.rb","lib/wmap/network_profiler.rb","lib/wmap/port_scanner.rb","lib/wmap/site_tracker.rb","lib/wmap/site_tracker/deactivated_site.rb","lib/wmap/url_checker.rb","lib/wmap/url_crawler.rb","lib/wmap/geoip_tracker.rb","lib/wmap/google_search_scraper.rb","lib/wmap/utils/logger.rb","lib/wmap/utils/domain_root.rb","lib/wmap/utils/url_magic.rb","lib/wmap/utils/utils.rb","lib/wmap/whois.rb","lib/wmap.rb","LICENSE.txt","version.txt","README.rdoc",  "wmap.gemspec"]
-  s.files += Dir['bin/*'] + Dir['demos/*'] + Dir['test/*'] + Dir['ruby_whois_patches/*'] + Dir['dicts/*'] + Dir['logs/wmap.log']
+  s.files = ["CHANGELOG.md", "TODO", "settings/discovery_ports","settings/google_keywords.txt","settings/google_locator.txt","data/","LICENSE.txt",
+							"version.txt","README.rdoc", "wmap.gemspec"]
+  s.files += Dir['lib/wmap/*.rb'] + Dir['lib/wmap/**/*.rb'] + Dir['bin/*'] + Dir['demos/*'] + Dir['test/*'] + Dir['ruby_whois_patches/*'] + Dir['dicts/*'] + Dir['logs/wmap.log']
   #s.homepage = "none"
   s.post_install_message = "*"*80 + "\n\nThank you for installing the wmap gem - a pure Ruby library for Internet web application discovery and tracking. Please refer to the README.rdoc for more information of using this gem.  \n\n" + "*"*80 + "\n"
   s.require_paths = ["lib"]
-  s.required_ruby_version = Gem::Requirement.new(">= 1.9.2")
+  s.required_ruby_version = Gem::Requirement.new(">= 2.1")
+
+ 	s.add_dependency 'dnsruby', '>= 1.52'
+	s.add_dependency 'geoip', '>= 1.0'
+	s.add_dependency 'minitest', '>= 5.0'
+	s.add_dependency 'net-ping', '>= 2.0'
+	s.add_dependency 'nokogiri', '>= 1.6'
+	s.add_dependency 'css_parser', '>= 1.6'
+	s.add_dependency 'openssl', '>= 2.0'
+	s.add_dependency 'parallel', '>= 1.0'
+	s.add_dependency 'whois', '>= 2.7'
+	s.add_dependency 'httpclient', '~> 2.0'
+	s.add_dependency 'open_uri_redirections', '>= 0.2'
+	s.add_dependency 'netaddr', '~> 1.5'
+
   s.rubyforge_project = "wmap"
   s.rubygems_version = "1.8.24"
   s.summary = "A pure Ruby web application and service discovery API."

metadata

@@ -1,15 +1,183 @@
 --- !ruby/object:Gem::Specification
 name: wmap
 version: !ruby/object:Gem::Version
-  version: 2.4.4
+  version: 2.4.5
 platform: ruby
 authors:
-- Yang Li
+- Sam (Yang) Li
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-16 00:00:00.000000000 Z
-dependencies: []
+date: 2018-11-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: dnsruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.52'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.52'
+- !ruby/object:Gem::Dependency
+  name: geoip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: net-ping
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: css_parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: openssl
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: whois
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: open_uri_redirections
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: netaddr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
 description: wmap is written to perform Internet web application / service discovery.
   The discovery results are designed to be automatically tracked by the software.
 email: yang.li@owasp.org
@@ -37,6 +205,7 @@ files:
 - LICENSE.txt
 - README.rdoc
 - TODO
+- bin/RHPG
 - bin/deprime
 - bin/distrust
 - bin/googleBot
@@ -45,6 +214,7 @@ files:
 - bin/run_tests
 - bin/spiderBot
 - bin/trust
+- bin/trusts
 - bin/updateAll
 - bin/wadd
 - bin/wadds
@@ -52,14 +222,8 @@ files:
 - bin/wdel
 - bin/wdump
 - bin/wmap
+- bin/wmaps
 - bin/wscan
-- data/cidrs
-- data/deactivated_sites
-- data/domains
-- data/hosts
-- data/prime_hosts
-- data/sites
-- data/sub_domains
 - demos/bruter.rb
 - demos/dns_brutes.rb
 - demos/filter_cidr.rb
@@ -88,7 +252,7 @@ files:
 - dicts/gtld.txt
 - dicts/hostnames-dict.big
 - dicts/hostnames-dict.txt
-- lib/wmap.rb
+- dicts/tlds.txt
 - lib/wmap/cidr_tracker.rb
 - lib/wmap/dns_bruter.rb
 - lib/wmap/domain_tracker.rb
@@ -108,6 +272,7 @@ files:
 - lib/wmap/utils/url_magic.rb
 - lib/wmap/utils/utils.rb
 - lib/wmap/whois.rb
+- lib/wmap/wp_tracker.rb
 - logs/wmap.log
 - ruby_whois_patches/base_cocca2.rb
 - ruby_whois_patches/kero.yachay.pe.rb
@@ -187,7 +352,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.2
+      version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -195,7 +360,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: wmap
-rubygems_version: 2.4.5.2
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: A pure Ruby web application and service discovery API.

data/data/cidrs

@@ -1,2 +0,0 @@
-# Local cidrs file created by Wmap::CidrTracker.save method at: 2015-07-07 12:23:56 -0400
-Network CIDR, CIDR RIPE Reference Text, CIDR NETNAME

data/data/deactivated_sites

@@ -1 +0,0 @@
-# Local site store created by class Wmap::SiteTracker::DeactivatedSite method save_sites_to_file! at: 2015-04-27 08:43:06 -0400

data/data/domains

@@ -1,2 +0,0 @@
-# Local domains file created by class Wmap::DomainTracker method save_domains_to_file! at: 2015-07-07 12:24:49 -0400
-# domain name, free zone transfer detected?

data/data/hosts

@@ -1 +0,0 @@
-# local hosts file created by the Wmap::HostTracker class save_known_hosts_to_file! method at: 2015-04-21 08:59:33 -0400

data/data/prime_hosts

@@ -1 +0,0 @@
-# local hosts file created by the Wmap::HostTracker::PrimaryHost class save_known_hosts_to_file! method at: 2015-07-07 12:25:34 -0400

data/data/sites

@@ -1,2 +0,0 @@
-# Local site store created by class Wmap::SiteTracker method save_sites_to_file! at: 2015-04-22 14:03:45 -0400
-# Website,Primary IP,Port,Hosting Status,Server,Response Code,Site MD5 Finger-print,Site Redirection,Timestamp

data/data/sub_domains

@@ -1,2 +0,0 @@
-# Local domains file created by class Wmap::DomainTracker::SubDomain method save_sub_domains_to_file! at: 2018-01-08 10:18:24 -0500
-# domain name, free zone transfer detected?

data/lib/wmap.rb

@@ -1,227 +0,0 @@
-#--
-# Wmap
-#
-# A pure Ruby library for the Internet web application discovery and tracking.
-#
-# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
-#++
-require 'wmap/utils/domain_root'
-require 'wmap/utils/url_magic'
-require 'wmap/utils/logger'
-require 'wmap/utils/utils'
-require 'wmap/cidr_tracker'
-require 'wmap/domain_tracker'
-require 'wmap/domain_tracker/sub_domain'
-require 'wmap/host_tracker'
-require 'wmap/host_tracker/primary_host'
-require 'wmap/whois'
-require 'wmap/url_checker'
-require 'wmap/network_profiler'
-require 'wmap/port_scanner'
-require 'wmap/url_crawler'
-require 'wmap/dns_bruter'
-require 'wmap/site_tracker'
-require 'wmap/site_tracker/deactivated_site'
-require 'wmap/geoip_tracker'
-require 'wmap/google_search_scraper'
-
-module Wmap
-
-  NAME            = "Wmap"
-  GEM             = "wmap"
-  VERSION		  = File.dirname(__FILE__) + "/../version.txt"
-
-  class << self
-	attr_accessor :known_internet_domains
-	attr_writer   :verbose
-
-	# Simple parser for the project version file
-	def read_ver
-		ver=Hash.new
-		f=File.open(VERSION,'r')
-		f.each do |line|
-			line.chomp!
-			case line
-			when /^(\s)*#/
-				next
-			when /\=/
-				entry=line.split("=").map! {|x| x.strip}
-				ver[entry[0]]=entry[1]
-			end
-		end
-		f.close
-		return ver
-	end
-
-	# Project banner in ASCII Art 'soft' format, courtesy to http://patorjk.com/software/taag/
-	def banner
-		ver=read_ver
-		art=",--.   ,--.       ,--.       ,--.   ,--.
-|  |   |  | ,---. |  |-.     |   `.'   | ,--,--. ,---.  ,---.  ,---. ,--.--.
-|  |.'.|  || .-. :| .-. '    |  |'.'|  |' ,-.  || .-. || .-. || .-. :|  .--'
-|   ,'.   |\   --.| `-' |    |  |   |  |\ '-'  || '-' '| '-' '\   --.|  |
-'--'   '--' `----' `---'     `--'   `--' `--`--'|  |-' |  |-'  `----'`--'
-                                                `--'   `--'                  "
-		string = "-"*80 + "\n" + art + "\n" + "Version: " + ver["version"] + "\tRelease Date: " + ver["date"] + "\nDesigned and developed by: " + ver["author"] + "\nEmail: " + ver["email"] + "\tLinkedIn: " + ver["linkedin"] + "\n" + "-"*80
-	end
-
-	# Explorer to discover and inventory web application / service automatically
-	def wmap(seed)
-		cmd="bin/wmap" + " " + seed
-		system(cmd)
-	end
-
-	# Crawler to search url contents for new sites
-	def crawl(url)
-		crawler=Wmap::UrlCrawler.new
-		crawler.crawl(url)
-	end
-
-	# whois query and sort the result into structured data
-	def whois(domain)
-		whois=Wmap::Whois.new(:verbose=>false)
-		whois.query(domain)
-	end
-
-	# Fast tcp port scanner on a single host or IP
-	def scan(host)
-		scanner=Wmap::PortScanner.new
-		scanner.scan(host)
-	end
-
-	# Fast multi-processes tcp port scanner on a list of targets
-	def scans(target_list)
-		scanner=Wmap::PortScanner.new
-		scanner.scans(target_list)
-	end
-
-	# CIDR Tracking - check the host against the local CIDR seed file, return the CIDR tracking path if found
-	def track(host)
-		tracker=Wmap::CidrTracker.new
-		tracker.cidr_worker(host)
-	end
-
-	# GeoIP Tracking - check the host / IP against the GeoIP data repository, return the Geographic information if found
-	def geoip(host)
-		tracker=Wmap::GeoIPTracker.new
-		tracker.query(host)
-	end
-
-	# URL checker - check the status of the remote URL
-	def check(url)
-		checker=Wmap::UrlChecker.new(:verbose=>false)
-		checker.url_worker(url)
-	end
-
-	# Check if the IP is within the range of the known CIDR blocks
-	def ip_trusted?(ip)
-		tracker=Wmap::CidrTracker.new
-		tracker.ip_trusted?(ip)
-	end
-
-	# Domain Tracking - check with the trust domain seed file locally, to determine if it's a new internet domain
-	# NOT to confuse with the Internet 'whois' lookup
-	def domain_known?(domain)
-		tracker=Wmap::DomainTracker.new
-		tracker.domain_known?(domain)
-	end
-
-	# Host Tracking - check local hosts file to see if this is a hostname known from the host seed file
-	# NOT to confuse with a regular DNS lookup over the internet
-	def host_known?(host)
-		tracker=Wmap::HostTracker.new.host_known?(host)
-	end
-
-	# Sub-domain tracking - check local hosts file to see if the sub-domain is already known
-	def sub_domain_known?(host)
-		tracker=Wmap::HostTracker.new.sub_domain_known?(host)
-	end
-
-	# IP Tracking - check local hosts file to see if this is an IP known from the seed file
-	# NOT to confuse with a regular reverse DNS lookup over the internet
-	def ip_known?(ip)
-		tracker=Wmap::HostTracker.new.ip_known?(ip)
-	end
-
-	# DNS Brute Forcer
-	def dns_brute(domain)
-		bruter=Wmap::DnsBruter.new
-		bruter.query(domain)
-	end
-
-	# Retrieve root domain from a host
-	def domain_root(host)
-		Wmap::Utils.get_domain_root(host)
-	end
-
-	# Log the information into file
-	def wlog(msg,agent,log_file)
-		Wmap::Utils.wlog(msg,agent,log_file)
-	end
-
-	# Host-name mutation for catch easily guessable hostname, i.e. "ww1.example.com" => ["ww1,example.com","ww2.example.com",...]
-	def mutation (host)
-		Wmap::DnsBruter.new.hostname_mutation(host)
-	end
-
-	# Check URL/Site response code
-	def response_code(url)
-		checker=Wmap::UrlChecker.new
-		checker.response_code(url)
-	end
-
-	# Search the site repository for all entries that match the pattern
-	def search(pattern)
-		searcher=Wmap::SiteTracker.new
-		searcher.search(pattern)
-	end
-
-	# Dump out the unique sites into a plain file
-	def dump(file)
-			store=Wmap::SiteTracker.new
-			store.save_uniq_sites(file)
-	end
-
-	# Dump out the unique sites into a XML file
-	def dump_xml(file)
-			store=Wmap::SiteTracker.new
-			store.save_uniq_sites_xml(file)
-	end
-
-	# Refresh the site information in the local data repository
-	def refresh(site)
-			store=Wmap::SiteTracker.new
-			store.refresh(site)
-			store.save!
-	end
-
-	# Refresh the site information in the local data repository
-	def refresh_all
-			store=Wmap::SiteTracker.new
-			store.refresh_all
-			store.save!
-	end
-
-	# Search the Google engines and sort out sites known by Google
-	def google
-		sites=Wmap::GoogleSearchScraper.new.workers.keys
-	end
-
-	# Print a site's full information from the repository
-	def print(site)
-		searcher=Wmap::SiteTracker.new
-		searcher.print_site(site)
-	end
-
-	# Print a site's full information from the repository
-	def print_all
-		searcher=Wmap::SiteTracker.new
-		searcher.print_all_sites
-	end
-
-  private
-
-
-
-  end
-end