wmap 2.8.1 → 2.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0430d7df339ee9c4f6099e1175b6cdef5ee0a453c2a5340e72e897e92a24ecfd
-  data.tar.gz: bb4aef610f1f71c992a25c41e0bcd1601039a224a250c251a4fc6b64bc2c362c
+  metadata.gz: 2dee2577809daef8231ddc3d97a7e0abab5312700f36ab2233366ff6729ae388
+  data.tar.gz: 5dee805b28f88e2ae320e6afd9b4ef7d6f25bff99a34cf46b42c32249bf7eaad
 SHA512:
-  metadata.gz: 371d69e9e72145befa5589a79ca7a7ac541c022881fc4a7ebf2c0ccedd3ab92962557487f7195f216427e1cc3f876efcbcae7cd00adf6ea4e55fec72ec94c5fe
-  data.tar.gz: f3e2245e00a680240b8c0b9f41a2074aff46f5b5ae7cd71ace31be71456a7ee0b909f95223b49dc846cd2f40bdd71cb26f815553943b2b1e78e9b2b567a8008c
+  metadata.gz: c78742dad1d356b88d2b45a0773527fd8f4e54cbff13c0ccb5f9fd9a228fac56cc2c26a705e8c5ce068b4d2bdf50d1fd190060501a1fc17e0a1040e583080db6
+  data.tar.gz: 7eea2eaa6ee45e9ac7857eaa603177b00424a6842cdd89a200062bdcd737c1a66fa2310645e80272fce71d0193fb9ca8044f232e0dabdf20c2da453e436c84cc

data/bin/wmap

@@ -13,7 +13,7 @@ parser = OptionParser.new do|opts|
 	opts.on('-d', '--data_dir data_dir', 'Web Mapper local cache data directory') do |data_dir|
 		options[:data_dir] = data_dir;
 	end
-	opts.on('-t', '--target target', 'Web Mapper target') do |target|
+	opts.on('-t', '--target target', 'Web Mapper target / seed for discovery') do |target|
 		options[:target] = target;
 	end
 	opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
@@ -133,6 +133,10 @@ Wmap.wlog(dis_urls.keys, "wmap", Log_dir+"discovered_urls.log") unless dis_urls.
 Wmap.wlog(dis_sites.keys, "wmap", Log_dir+"discovered_sites.log") unless dis_sites.empty?
 #crawler.wlog(c_start.keys,Log_dir+"crawler.log")
 #crawler.wlog(c_done.keys,Log_dir+"crawler.log")
+
+
+# Save the current disovery urls only to a specific file, patched 07/23/2021
+crawler.save_discovered_urls(Log_dir+"cur_urls.log")
 crawler=nil
 
 

data/lib/wmap/url_crawler.rb

@@ -43,7 +43,7 @@ class Wmap::UrlCrawler
 		@crawl_start=Hash.new
 		@crawl_done=Hash.new
 		Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
-		@log_dir=@data_dir + "/../logs/"
+		@log_dir=@data_dir + "/logs/"
 		Dir.mkdir(@log_dir) unless Dir.exist?(@log_dir)
 		@log_file=@log_dir + "crawler.log"
 	end

data/version.txt

@@ -3,8 +3,8 @@
 ###############################################################################
 package = wmap
 # wmap version 2.0 == web_discovery version 1.5.3
-version = 2.8.1
-date = 2020-05-08
+version = 2.8.2
+date = 2021-07-23
 
 author = Sam (Yang) Li
 email = yang.li@owasp.org

data/wmap.gemspec

@@ -36,7 +36,7 @@ Gem::Specification.new do |s|
   s.description = "wmap is written to perform Internet web application / service discovery. The discovery results are designed to be automatically tracked by the software."
   s.email = info["email"]
   s.executables = ["wmap","wscan","wadd","wadds","wdel","wcheck","wdump","spiderBot","googleBot","updateAll","prime","deprime","refresh","trust","trusts","distrust","run_tests"]
-  s.files = ["CHANGELOG.md", "TODO", "settings/discovery_ports","data/","LICENSE.txt",
+  s.files = ["CHANGELOG.md", "TODO", "settings/discovery_ports", "LICENSE.txt",
 							"version.txt","README.md", "wmap.gemspec"]
   s.files += Dir['lib/*.rb'] + Dir['lib/wmap/*.rb'] + Dir['lib/wmap/**/*'] + Dir['bin/*'] + Dir['settings/*'] + Dir['demos/*'] + Dir['test/*'] + Dir['dicts/*']
   #s.homepage = "none"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wmap
 version: !ruby/object:Gem::Version
-  version: 2.8.1
+  version: 2.8.2
 platform: ruby
 authors:
 - Sam (Yang) Li
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-05-08 00:00:00.000000000 Z
+date: 2021-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dnsruby
@@ -234,7 +234,6 @@ files:
 - LICENSE.txt
 - README.md
 - TODO
-- bin/RHPG
 - bin/deprime
 - bin/distrust
 - bin/googleBot
@@ -251,7 +250,6 @@ files:
 - bin/wdel
 - bin/wdump
 - bin/wmap
-- bin/wmaps
 - bin/wscan
 - demos/bruter.rb
 - demos/dns_brutes.rb
@@ -308,7 +306,6 @@ files:
 - settings/discovery_ports
 - settings/google_keywords.txt
 - settings/google_locator.txt
-- settings/tag_signatures
 - test/cidr_tracker_test.rb
 - test/domain_tracker_test.rb
 - test/utils_test.rb
@@ -336,9 +333,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: wmap
-rubygems_version: 2.7.10
-signing_key: 
+rubygems_version: 3.0.9
+signing_key:
 specification_version: 4
 summary: A pure Ruby web application and service discovery API.
 test_files: []

data/bin/RHPG

@@ -1,107 +0,0 @@
-#!/usr/bin/env ruby
-# Executable to lookup then merge site tech details into the RHPG asset spreadsheet in CSV format only
-#
-## Usage: RHPG [RHPG.csv]
-require "wmap"
-require "csv"
-include Wmap::Utils
-
-def print_usage
-	puts "Program to lookup then merge the site details into RHPG asset spreadsheet. \nUsage: RHPG [RHPG.csv]"
-end
-
-# Lookup the site store for a domain; then return the fingger print info of the site
-def site_tracker_lookup(domain)
-  tracker=Wmap::SiteTracker.instance
-	tracker.verbose=false
-  #first order search
-  tracker.known_sites.each do |key,val|
-    if key.include?(domain.strip.downcase) && key.include?("https")
-      tracker=nil
-      return [key] + val.values
-    end
-  end
-  #second order search
-  tracker.known_sites.each do |key,val|
-    if key.include?(domain.strip.downcase)
-      tracker=nil
-      return [key] + val.values
-    end
-  end
-  tracker=nil
-  return [nil]*9
-end
-
-# look up the wp site data store for a domain; then return the wp finger print info: [is_wp?,wp_ver]
-def wp_tracker_lookup(domain)
-  tracker=Wmap::WpTracker.new(:verbose=>false)
-	# first order
-	tracker.known_wp_sites.each do |key,val|
-		if key.include?(domain.strip.downcase) && val
-			ver=tracker.wp_ver(key)
-			tracker=nil
-			return [val,ver]
-		end
-	end
-  # second order
-  tracker.known_wp_sites.each do |key,val|
-    if key.include?(domain.strip.downcase) && key.include?("https") && val
-      tracker=nil
-      return [val,nil]
-    end
-  end
-  # third order
-  tracker.known_wp_sites.each do |key,val|
-    if key.include?(domain.strip.downcase)
-      tracker=nil
-      return [val,nil]
-    end
-  end
-  tracker=nil
-  return [nil,nil]
-end
-
-# perform the wpscan on a site
-def wpscan(domain)
-	url=site_tracker_lookup(domain)[0]
-	return nil if url.nil?
-	if url.include?("https")
-		command="wpscan --disable-tls-checks --ignore-main-redirect --url=" + url + " -o " + domain + ".wpscan"
-	else
-		command="wpscan --ignore-main-redirect --url=" + url + " -o " + domain + ".wpscan"
-	end
-	system(command)
-end
-
-puts Wmap.banner
-print_usage
-
-# open output file to write
-CSV.open("output.csv", "wb") do |csv|
-  cnt=1
-  # open RHPG input file to read
-  CSV.foreach(ARGV[0]) do |row|
-    puts "Processing row #{cnt}"
-    #puts row.inspect
-    my_row=Array.new
-    if cnt > 1
-      if is_domain?(row[0])
-=begin
-				if row[3] =~ /Keep/i && row[3] != /Redirect/i
-					unless File.exist?(row[0]+".wpscan")
-						wpscan(row[0])
-					end
-				end
-=end
-        my_row = row + site_tracker_lookup(row[0]) + wp_tracker_lookup(row[0])
-      else
-        my_row = row + [nil]*10
-      end
-    else
-      my_row = row + ["Website","Primary IP","Port","Hosting Status","Server","Response Code","MD5 Finger-print","Redirection","Timestamp", "WordPress", "WordPress Version"]
-    end
-    cnt+=1
-    csv << my_row
-  end
-  puts "All done. "
-end

data/bin/wmaps

@@ -1,23 +0,0 @@
-#!/usr/bin/env ruby
-# script to automate the new site discovery through by crawling all unique sites in the site store
-require "wmap"
-require "parallel"
-
-def wmap_worker(domain)
-   cmd = "wmap " + domain
-   puts "wmap discovery on domain: ", domain
-   system(cmd)
-end
-
-
-tracker=Wmap::DomainTracker.instance
-Parallel.map(tracker.known_internet_domains.keys, :in_processes => 10) { |target|
-  puts "Working on #{target} ..." if @verbose
-  wmap_worker(target)
-}
-=begin
-tracker.known_internet_domains.keys.map do |domain|
-  wmap_worker(domain)
-end
-=end
-tracker=nil

data/settings/tag_signatures

@@ -1,6 +0,0 @@
-# Adware signature file: signture string, description
-gtag.js, Google / DoubleClick Floodlight Tag
-analytics.js, Google Universal Analytics Tag
-ga.js, Google Analytics Tag
-utag.js, Tealium Tag
-all.js, Facebook Tag