wmap 2.4.5 → 2.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 781e42205ee21710dc610778284fb6ab3fc4644598632d204db6bdc291af216f
-  data.tar.gz: c2c1c527be70b2452b1aa66ea2138af78a094ae82eb8ddfde15a2ec09df30700
+  metadata.gz: 7fd34fa6e6a86b6d7a4f2eb0d232023832473ed436870ddab082aa9768cb4bc2
+  data.tar.gz: 7a8e8d44eae5cf2a7c51077cfd7dcfa73a49048833758814d35378438c546195
 SHA512:
-  metadata.gz: ab3231e48a2d65f777afa36a9efb28bc741dfe79f0a10aae64d18ad6861206ca209984635b57ffe863bfa4a602d07cc79ad472561c3eccf1fb527973ef15bd0b
-  data.tar.gz: 55ead86d6498b73e0b2511518dcd3cdc69468456fc3525ff523b03a38b31bbadd18185e94c8afea777c3dd3f0d943b7ee1868482bbd2badecb4b339ad14eb6f5
+  metadata.gz: 80071d5b153195da339981c4abdc175d4b4368870dd85da822ab2b84fdc66b32e0e7fafcb4b99ab909309f2b90bf671dffb1b5da0a282ea35cc01851c4cf2430
+  data.tar.gz: d8fa25c38b2a147863184a6feca7b94e0588798394046d574030d394b34d9c26ddc5e851539b7ec154de5aa6467660b2eb65cc72137c61d01888d72d23af481d

data/CHANGELOG.md

@@ -11,6 +11,7 @@
 
 ## Mile-stones
 
+- November 2018: Pick up the maintenances.
 - July 2015: Move the project under OWASP
 - November 2014: Re-name from web_discovery to wmap, re-factor the code base to better scale up
 	across the board

data/README.rdoc

@@ -6,12 +6,12 @@ This program is designed for the web application asset discovery and tracking. I
 to cover the gaps of a similar commercial product. Over the time it grows to be a more capable and complete replacement (IMHO).
 
 
-== Wmap in Motion
-Use the demo web app build on top of wmap gem: http://wmap.io/
+== WMAP in Motion
+Use the demo web app build on top of wmap gem: http://wmap.io
 
 
 == Program Version
-The latest release is Beta version 1.5.x as of fall 2014. Please refer to the CHANGELOG.md for the program's history information.
+The latest release is version 2.4.5. as of fall 2018. Please refer to the CHANGELOG.md for the program's history information.
 
 
 == Author Contact
@@ -19,15 +19,15 @@ This program is designed and developed by Yang Li. You can reach him by Email: <
 
 
 == Installation
-To take full power of this program, you would need an *nix flavor machine with direct Internet access. I have installed it successfully on both Mac and Linux machines. You'll also need the Ruby environment being setup properly. The easiest way to install OWASP Web Mapper is by using Ruby Gems. Download the latest gem 'wmap-x.x.x.gem' into the local file system. Then install it from command line there:
+To take full power of this program, you would need an *nix flavor machine with direct Internet access. I have installed it successfully on both Mac and Linux machines. You'll also need the Ruby environment being setup properly. The easiest way to install OWASP Web Mapper is by using Ruby Gems. You can install it from command line:
 
-  gem install wmap-x.x.x.gem --no-rdoc
+  gem install wmap
 
 == Specific Installation Problem with Nokogiri
 Nokogiri is a native xml/html parser used by the project. It's fast and powerful. However, it comes with pitfall of installation problem around building native extension for your environment. Please refer to this page for trouble-shooting tip (http://www.nokogiri.org/tutorials/installing_nokogiri.html).
 
 == Dependency
-You need the Ruby 1.9.2 or above in order to use this program. In my test environment, I was able to set it up with RVM. Please refer to this page for more installation information: http://www.ruby-lang.org/en/downloads/
+You need the Ruby 2.1.0 or above in order to use this program. In my test environment, I was able to set it up with <a href="https://rvm.io/">RVM</a>. Please refer to this page for more installation information: https://www.ruby-lang.org/en/documentation/installation/
 
 In addition, the following Ruby GEM dependency are needed by different components of this software. The should be installed automatically:
       require "dnsruby"

data/lib/wmap.rb

@@ -0,0 +1,228 @@
+#--
+# Wmap
+#
+# A pure Ruby library for the Internet web application discovery and tracking.
+#
+# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
+#++
+require 'wmap/utils/domain_root'
+require 'wmap/utils/url_magic'
+require 'wmap/utils/logger'
+require 'wmap/utils/utils'
+require 'wmap/cidr_tracker'
+require 'wmap/domain_tracker'
+require 'wmap/domain_tracker/sub_domain'
+require 'wmap/host_tracker'
+require 'wmap/host_tracker/primary_host'
+require 'wmap/whois'
+require 'wmap/url_checker'
+require 'wmap/wp_tracker'
+require 'wmap/network_profiler'
+require 'wmap/port_scanner'
+require 'wmap/url_crawler'
+require 'wmap/dns_bruter'
+require 'wmap/site_tracker'
+require 'wmap/site_tracker/deactivated_site'
+require 'wmap/geoip_tracker'
+require 'wmap/google_search_scraper'
+
+module Wmap
+
+  NAME            = "Wmap"
+  GEM             = "wmap"
+  VERSION		  = File.dirname(__FILE__) + "/../version.txt"
+
+  class << self
+	attr_accessor :known_internet_domains
+	attr_writer   :verbose
+
+	# Simple parser for the project version file
+	def read_ver
+		ver=Hash.new
+		f=File.open(VERSION,'r')
+		f.each do |line|
+			line.chomp!
+			case line
+			when /^(\s)*#/
+				next
+			when /\=/
+				entry=line.split("=").map! {|x| x.strip}
+				ver[entry[0]]=entry[1]
+			end
+		end
+		f.close
+		return ver
+	end
+
+	# Project banner in ASCII Art 'soft' format, courtesy to http://patorjk.com/software/taag/
+	def banner
+		ver=read_ver
+		art=",--.   ,--.       ,--.       ,--.   ,--.
+|  |   |  | ,---. |  |-.     |   `.'   | ,--,--. ,---.  ,---.  ,---. ,--.--.
+|  |.'.|  || .-. :| .-. '    |  |'.'|  |' ,-.  || .-. || .-. || .-. :|  .--'
+|   ,'.   |\   --.| `-' |    |  |   |  |\ '-'  || '-' '| '-' '\   --.|  |
+'--'   '--' `----' `---'     `--'   `--' `--`--'|  |-' |  |-'  `----'`--'
+                                                `--'   `--'                  "
+		string = "-"*80 + "\n" + art + "\n" + "Version: " + ver["version"] + "\tRelease Date: " + ver["date"] + "\nDesigned and developed by: " + ver["author"] + "\nEmail: " + ver["email"] + "\tLinkedIn: " + ver["linkedin"] + "\n" + "-"*80
+	end
+
+	# Explorer to discover and inventory web application / service automatically
+	def wmap(seed)
+		cmd="bin/wmap" + " " + seed
+		system(cmd)
+	end
+
+	# Crawler to search url contents for new sites
+	def crawl(url)
+		crawler=Wmap::UrlCrawler.new
+		crawler.crawl(url)
+	end
+
+	# whois query and sort the result into structured data
+	def whois(domain)
+		whois=Wmap::Whois.new(:verbose=>false)
+		whois.query(domain)
+	end
+
+	# Fast tcp port scanner on a single host or IP
+	def scan(host)
+		scanner=Wmap::PortScanner.new
+		scanner.scan(host)
+	end
+
+	# Fast multi-processes tcp port scanner on a list of targets
+	def scans(target_list)
+		scanner=Wmap::PortScanner.new
+		scanner.scans(target_list)
+	end
+
+	# CIDR Tracking - check the host against the local CIDR seed file, return the CIDR tracking path if found
+	def track(host)
+		tracker=Wmap::CidrTracker.new
+		tracker.cidr_worker(host)
+	end
+
+	# GeoIP Tracking - check the host / IP against the GeoIP data repository, return the Geographic information if found
+	def geoip(host)
+		tracker=Wmap::GeoIPTracker.new
+		tracker.query(host)
+	end
+
+	# URL checker - check the status of the remote URL
+	def check(url)
+		checker=Wmap::UrlChecker.new(:verbose=>false)
+		checker.url_worker(url)
+	end
+
+	# Check if the IP is within the range of the known CIDR blocks
+	def ip_trusted?(ip)
+		tracker=Wmap::CidrTracker.new
+		tracker.ip_trusted?(ip)
+	end
+
+	# Domain Tracking - check with the trust domain seed file locally, to determine if it's a new internet domain
+	# NOT to confuse with the Internet 'whois' lookup
+	def domain_known?(domain)
+		tracker=Wmap::DomainTracker.new
+		tracker.domain_known?(domain)
+	end
+
+	# Host Tracking - check local hosts file to see if this is a hostname known from the host seed file
+	# NOT to confuse with a regular DNS lookup over the internet
+	def host_known?(host)
+		tracker=Wmap::HostTracker.new.host_known?(host)
+	end
+
+	# Sub-domain tracking - check local hosts file to see if the sub-domain is already known
+	def sub_domain_known?(host)
+		tracker=Wmap::HostTracker.new.sub_domain_known?(host)
+	end
+
+	# IP Tracking - check local hosts file to see if this is an IP known from the seed file
+	# NOT to confuse with a regular reverse DNS lookup over the internet
+	def ip_known?(ip)
+		tracker=Wmap::HostTracker.new.ip_known?(ip)
+	end
+
+	# DNS Brute Forcer
+	def dns_brute(domain)
+		bruter=Wmap::DnsBruter.new
+		bruter.query(domain)
+	end
+
+	# Retrieve root domain from a host
+	def domain_root(host)
+		Wmap::Utils.get_domain_root(host)
+	end
+
+	# Log the information into file
+	def wlog(msg,agent,log_file)
+		Wmap::Utils.wlog(msg,agent,log_file)
+	end
+
+	# Host-name mutation for catch easily guessable hostname, i.e. "ww1.example.com" => ["ww1,example.com","ww2.example.com",...]
+	def mutation (host)
+		Wmap::DnsBruter.new.hostname_mutation(host)
+	end
+
+	# Check URL/Site response code
+	def response_code(url)
+		checker=Wmap::UrlChecker.new
+		checker.response_code(url)
+	end
+
+	# Search the site repository for all entries that match the pattern
+	def search(pattern)
+		searcher=Wmap::SiteTracker.new
+		searcher.search(pattern)
+	end
+
+	# Dump out the unique sites into a plain file
+	def dump(file)
+			store=Wmap::SiteTracker.new(:verbose=>true)
+			store.save_uniq_sites(file)
+	end
+
+	# Dump out the unique sites into a XML file
+	def dump_xml(file)
+			store=Wmap::SiteTracker.new
+			store.save_uniq_sites_xml(file)
+	end
+
+	# Refresh the site information in the local data repository
+	def refresh(site)
+			store=Wmap::SiteTracker.new
+			store.refresh(site)
+			store.save!
+	end
+
+	# Refresh the site information in the local data repository
+	def refresh_all
+			store=Wmap::SiteTracker.new
+			store.refresh_all
+			store.save!
+	end
+
+	# Search the Google engines and sort out sites known by Google
+	def google
+		sites=Wmap::GoogleSearchScraper.new.workers.keys
+	end
+
+	# Print a site's full information from the repository
+	def print(site)
+		searcher=Wmap::SiteTracker.new
+		searcher.print_site(site)
+	end
+
+	# Print a site's full information from the repository
+	def print_all
+		searcher=Wmap::SiteTracker.new
+		searcher.print_all_sites
+	end
+
+  private
+
+
+
+  end
+end

data/lib/wmap/cidr_tracker.rb

@@ -17,6 +17,7 @@ class Wmap::CidrTracker
 	def initialize (params = {})
 		@verbose=params.fetch(:verbose, false)
 		@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
+		Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
 		@file_cidr_seeds=params.fetch(:cidr_seeds, @data_dir + 'cidrs')
 		@known_cidr_blks={}
 		@known_cidr_blks_desc_index=[]

data/lib/wmap/dns_bruter.rb

@@ -20,6 +20,7 @@ class Wmap::DnsBruter
 	def initialize (params = {})
 		# Change to your brute-force dictionary file here if necessary
 		@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
+		Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
 		@file_hosts = @data_dir + 'hosts'
 		@file_hosts_dict = File.dirname(__FILE__)+'/../../dicts/hostnames-dict.txt'
 

data/lib/wmap/domain_tracker.rb

@@ -23,6 +23,7 @@ class Wmap::DomainTracker
 		# Initialize the instance variables
 		@verbose=params.fetch(:verbose, false)
 		@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
+		Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
 		@file_domains=params.fetch(:domains_file, @data_dir+'domains')
 		@max_parallel=params.fetch(:max_parallel, 40)
 		# Hash table to hold the trusted domains

data/lib/wmap/host_tracker.rb

@@ -21,6 +21,7 @@ class Wmap::HostTracker
 	def initialize (params = {})
 		@verbose=params.fetch(:verbose, false)
 		@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
+		Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
 		# Set default instance variables
 		@file_hosts=@data_dir + 'hosts'
 		file=params.fetch(:hosts_file, @file_hosts)

data/lib/wmap/site_tracker.rb

@@ -22,6 +22,7 @@ class Wmap::SiteTracker
 	def initialize (params = {})
 		# Initialize the instance variables
 		@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
+		Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
 		@file_sites=@data_dir+'sites'
 		@file_stores=params.fetch(:sites_file, @file_sites)
 		@verbose=params.fetch(:verbose, false)

data/lib/wmap/wp_tracker.rb

@@ -27,6 +27,7 @@ class Wmap::WpTracker
 	def initialize (params = {})
 		@verbose=params.fetch(:verbose, false)
 		@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
+		Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
     wp_sites=@data_dir+'wp_sites'
     @file_wps=params.fetch(:sites_wp, wp_sites)
 		@http_timeout=params.fetch(:http_timeout, 5000)

data/version.txt

@@ -3,8 +3,8 @@
 ###############################################################################
 package = wmap
 # wmap version 2.0 == web_discovery version 1.5.3
-version = 2.4.5
-date = 2018-11-06
+version = 2.4.6
+date = 2018-11-19
 
 author = Sam (Yang) Li
 email = yang.li@owasp.org

data/wmap.gemspec

@@ -38,7 +38,7 @@ Gem::Specification.new do |s|
   s.executables = ["wmap","wscan","wadd","wadds","wdel","wcheck","wdump","spiderBot","googleBot","updateAll","prime","deprime","refresh","trust","distrust","run_tests"]
   s.files = ["CHANGELOG.md", "TODO", "settings/discovery_ports","settings/google_keywords.txt","settings/google_locator.txt","data/","LICENSE.txt",
 							"version.txt","README.rdoc", "wmap.gemspec"]
-  s.files += Dir['lib/wmap/*.rb'] + Dir['lib/wmap/**/*.rb'] + Dir['bin/*'] + Dir['demos/*'] + Dir['test/*'] + Dir['ruby_whois_patches/*'] + Dir['dicts/*'] + Dir['logs/wmap.log']
+  s.files += Dir['lib/*.rb'] + Dir['lib/wmap/*.rb'] + Dir['lib/wmap/**/*.rb'] + Dir['bin/*'] + Dir['demos/*'] + Dir['test/*'] + Dir['ruby_whois_patches/*'] + Dir['dicts/*'] + Dir['logs/wmap.log']
   #s.homepage = "none"
   s.post_install_message = "*"*80 + "\n\nThank you for installing the wmap gem - a pure Ruby library for Internet web application discovery and tracking. Please refer to the README.rdoc for more information of using this gem.  \n\n" + "*"*80 + "\n"
   s.require_paths = ["lib"]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wmap
 version: !ruby/object:Gem::Version
-  version: 2.4.5
+  version: 2.4.6
 platform: ruby
 authors:
 - Sam (Yang) Li
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-06 00:00:00.000000000 Z
+date: 2018-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dnsruby
@@ -253,6 +253,7 @@ files:
 - dicts/hostnames-dict.big
 - dicts/hostnames-dict.txt
 - dicts/tlds.txt
+- lib/wmap.rb
 - lib/wmap/cidr_tracker.rb
 - lib/wmap/dns_bruter.rb
 - lib/wmap/domain_tracker.rb
@@ -360,7 +361,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: wmap
-rubygems_version: 2.7.7
+rubygems_version: 2.7.8
 signing_key: 
 specification_version: 4
 summary: A pure Ruby web application and service discovery API.