wmap 2.7.6 → 2.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1f2c900ee2ac330d411f3f6496868ea519dd322b236cd00be0b5f3d1d8d43db
-  data.tar.gz: 054a534426e0b9b99730ac6252a9b0e2424866df914d02f6f7f1bdb925588bc1
+  metadata.gz: 2dee2577809daef8231ddc3d97a7e0abab5312700f36ab2233366ff6729ae388
+  data.tar.gz: 5dee805b28f88e2ae320e6afd9b4ef7d6f25bff99a34cf46b42c32249bf7eaad
 SHA512:
-  metadata.gz: c591b085614ec050ff8637750dfff57c3238ccf42c2fce9cab985ccd0f3e4761ed085632a9db7f4af87be39e2921b33d330c890c1af87213a134bdd37f43553d
-  data.tar.gz: 414720066fee25c30b41beb9c7529f7bfffda513e01e1bf4445cfefcd9ecbf2c421aa64db931a4d489845d664705ededb2474c3240750ad12cd2423b21078987
+  metadata.gz: c78742dad1d356b88d2b45a0773527fd8f4e54cbff13c0ccb5f9fd9a228fac56cc2c26a705e8c5ce068b4d2bdf50d1fd190060501a1fc17e0a1040e583080db6
+  data.tar.gz: 7eea2eaa6ee45e9ac7857eaa603177b00424a6842cdd89a200062bdcd737c1a66fa2310645e80272fce71d0193fb9ca8044f232e0dabdf20c2da453e436c84cc

data/bin/wmap

@@ -13,7 +13,7 @@ parser = OptionParser.new do|opts|
 	opts.on('-d', '--data_dir data_dir', 'Web Mapper local cache data directory') do |data_dir|
 		options[:data_dir] = data_dir;
 	end
-	opts.on('-t', '--target target', 'Web Mapper target') do |target|
+	opts.on('-t', '--target target', 'Web Mapper target / seed for discovery') do |target|
 		options[:target] = target;
 	end
 	opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
@@ -29,6 +29,10 @@ parser.parse!
 # print program banner
 puts Wmap.banner
 # print_usage unless options[:target]
+unless options[:target]
+	puts "Usage:   $ wmap -h"
+	exit 1
+end
 
 # Preparing - check out the working logs directory
 if options[:data_dir]
@@ -129,6 +133,10 @@ Wmap.wlog(dis_urls.keys, "wmap", Log_dir+"discovered_urls.log") unless dis_urls.
 Wmap.wlog(dis_sites.keys, "wmap", Log_dir+"discovered_sites.log") unless dis_sites.empty?
 #crawler.wlog(c_start.keys,Log_dir+"crawler.log")
 #crawler.wlog(c_done.keys,Log_dir+"crawler.log")
+
+
+# Save the current disovery urls only to a specific file, patched 07/23/2021
+crawler.save_discovered_urls(Log_dir+"cur_urls.log")
 crawler=nil
 
 

data/dicts/tlds.txt

@@ -1,5 +1,5 @@
 # http://data.iana.org/TLD/tlds-alpha-by-domain.txt
-# Version 2018110500, Last Updated Mon Nov  5 07:07:02 2018 UTC
+# Version 2020033000, Last Updated Mon Mar 30 07:07:01 2020 UTC
 AAA
 AARP
 ABARTH
@@ -16,7 +16,6 @@ ACCENTURE
 ACCOUNTANT
 ACCOUNTANTS
 ACO
-ACTIVE
 ACTOR
 AD
 ADAC
@@ -146,7 +145,6 @@ BIZ
 BJ
 BLACK
 BLACKFRIDAY
-BLANCO
 BLOCKBUSTER
 BLOG
 BLOOMBERG
@@ -155,7 +153,6 @@ BM
 BMS
 BMW
 BN
-BNL
 BNPPARIBAS
 BO
 BOATS
@@ -214,7 +211,6 @@ CARE
 CAREER
 CAREERS
 CARS
-CARTIER
 CASA
 CASE
 CASEIH
@@ -247,7 +243,6 @@ CHEAP
 CHINTAI
 CHRISTMAS
 CHROME
-CHRYSLER
 CHURCH
 CI
 CIPRIANI
@@ -299,6 +294,7 @@ COUNTRY
 COUPON
 COUPONS
 COURSES
+CPA
 CR
 CREDIT
 CREDITCARD
@@ -360,9 +356,7 @@ DNP
 DO
 DOCS
 DOCTOR
-DODGE
 DOG
-DOHA
 DOMAINS
 DOT
 DOWNLOAD
@@ -371,7 +365,6 @@ DTV
 DUBAI
 DUCK
 DUNLOP
-DUNS
 DUPONT
 DURBAN
 DVAG
@@ -392,7 +385,6 @@ ENERGY
 ENGINEER
 ENGINEERING
 ENTERPRISES
-EPOST
 EPSON
 EQUIPMENT
 ER
@@ -408,7 +400,6 @@ EU
 EUROVISION
 EUS
 EVENTS
-EVERBANK
 EXCHANGE
 EXPERT
 EXPOSED
@@ -488,6 +479,7 @@ GAME
 GAMES
 GAP
 GARDEN
+GAY
 GB
 GBIZ
 GD
@@ -580,7 +572,6 @@ HOMEGOODS
 HOMES
 HOMESENSE
 HONDA
-HONEYWELL
 HORSE
 HOSPITAL
 HOST
@@ -634,7 +625,6 @@ IQ
 IR
 IRISH
 IS
-ISELECT
 ISMAILI
 IST
 ISTANBUL
@@ -699,12 +689,10 @@ KYOTO
 KZ
 LA
 LACAIXA
-LADBROKES
 LAMBORGHINI
 LAMER
 LANCASTER
 LANCIA
-LANCOME
 LAND
 LANDROVER
 LANXESS
@@ -725,7 +713,6 @@ LEGO
 LEXUS
 LGBT
 LI
-LIAISON
 LIDL
 LIFE
 LIFEINSURANCE
@@ -744,6 +731,7 @@ LIVING
 LIXIL
 LK
 LLC
+LLP
 LOAN
 LOANS
 LOCKER
@@ -819,7 +807,6 @@ MN
 MO
 MOBI
 MOBILE
-MOBILY
 MODA
 MOE
 MOI
@@ -827,7 +814,6 @@ MOM
 MONASH
 MONEY
 MONSTER
-MOPAR
 MORMON
 MORTGAGE
 MOSCOW
@@ -835,7 +821,6 @@ MOTO
 MOTORCYCLES
 MOV
 MOVIE
-MOVISTAR
 MP
 MQ
 MR
@@ -854,7 +839,6 @@ MY
 MZ
 NA
 NAB
-NADEX
 NAGOYA
 NAME
 NATIONWIDE
@@ -955,7 +939,6 @@ PHOTO
 PHOTOGRAPHY
 PHOTOS
 PHYSIO
-PIAGET
 PICS
 PICTET
 PICTURES
@@ -1152,18 +1135,16 @@ SONG
 SONY
 SOY
 SPACE
-SPIEGEL
 SPORT
 SPOT
 SPREADBETTING
 SR
 SRL
-SRT
+SS
 ST
 STADA
 STAPLES
 STAR
-STARHUB
 STATEBANK
 STATEFARM
 STC
@@ -1211,7 +1192,6 @@ TEAM
 TECH
 TECHNOLOGY
 TEL
-TELEFONICA
 TEMASEK
 TENNIS
 TEVA
@@ -1271,7 +1251,6 @@ TZ
 UA
 UBANK
 UBS
-UCONNECT
 UG
 UK
 UNICOM
@@ -1305,7 +1284,6 @@ VIP
 VIRGIN
 VISA
 VISION
-VISTAPRINT
 VIVA
 VIVO
 VLAANDEREN
@@ -1324,7 +1302,6 @@ WALMART
 WALTER
 WANG
 WANGGOU
-WARMAN
 WATCH
 WATCHES
 WEATHER
@@ -1452,13 +1429,14 @@ XN--MGBA7C0BBN0A
 XN--MGBAAKC7DVF
 XN--MGBAAM7A8H
 XN--MGBAB2BD
+XN--MGBAH1A3HJKRD
 XN--MGBAI9AZGQP6J
 XN--MGBAYH7GPA
-XN--MGBB9FBPOB
 XN--MGBBH1A
 XN--MGBBH1A71E
 XN--MGBC0A9AZCG
 XN--MGBCA7DZDO
+XN--MGBCPQ6GPA1A
 XN--MGBERP4A5D4AR
 XN--MGBGU82A
 XN--MGBI4ECEXP
@@ -1484,8 +1462,10 @@ XN--P1AI
 XN--PBT977C
 XN--PGBS0DH
 XN--PSSY2U
+XN--Q7CE6A
 XN--Q9JYB4C
 XN--QCKA1PMC
+XN--QXA6A
 XN--QXAM
 XN--RHQV96G
 XN--ROVU88B
@@ -1530,7 +1510,6 @@ ZAPPOS
 ZARA
 ZERO
 ZIP
-ZIPPO
 ZM
 ZONE
 ZUERICH

data/lib/wmap.rb

@@ -8,6 +8,7 @@
 require 'wmap/utils/domain_root'
 require 'wmap/utils/url_magic'
 require 'wmap/utils/logger'
+require 'wmap/utils/wp_detect'
 require 'wmap/utils/utils'
 require 'wmap/cidr_tracker'
 require 'wmap/domain_tracker'

data/lib/wmap/cidr_tracker.rb

@@ -99,8 +99,8 @@ class Wmap::CidrTracker
 		#@known_cidr_blks_asce_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>false)
 		@known_cidr_blks_asce_index=@known_cidr_blks.keys.sort
 		@known_cidr_blks_desc_index=@known_cidr_blks_asce_index.reverse
-	#rescue => ee
-	#	puts "Exception on method #{__method__}: #{ee}" # if @verbose
+	rescue => ee
+		puts "Exception on method #{__method__}: #{ee}" # if @verbose
 	end
 
 	# 'setter' to remove an entry to CIDR store @known_cidr_blks
@@ -167,6 +167,7 @@ class Wmap::CidrTracker
 			known = cidr4.contains?(ip+'/32')
 			break if known
 		end
+		return known
 	rescue => ee
 		puts "Exception on method #{__method__}: #{ee}" if @verbose
 		return false

data/lib/wmap/site_tracker/wp_tracker.rb

@@ -7,16 +7,14 @@
 #++
 require "parallel"
 #require "singleton"
-require "open-uri"
-require "open_uri_redirections"
-require "nokogiri"
-require "css_parser"
+
 
 module Wmap
 class SiteTracker
 
 class WpTracker < Wmap::SiteTracker
 	include Wmap::Utils
+	include Wmap::Utils::WpDetect
 	#include Singleton
 
 	attr_accessor :http_timeout, :max_parallel, :verbose, :sites_wp, :data_dir
@@ -152,27 +150,6 @@ class WpTracker < Wmap::SiteTracker
 	end
 	alias_method :adds, :bulk_add
 
-  # logic to determin if it's a wordpress site
-  def is_wp?(url)
-		site=url_2_site(url)
-		if wp_readme?(site)
-			found=true
-		elsif wp_css?(site)
-			found=true
-		elsif wp_meta?(site)
-			found=true
-		elsif wp_login?(site)
-			found=true
-		elsif wp_rpc?(site)
-			found=true
-		else
-			found=false
-		end
-		return found
-	rescue => ee
-		puts "Exception on method #{__method__}: #{ee}: #{url}" if @verbose
-	end
-
 	# Refresh one site entry then update the instance variable (cache)
 	def refresh (target,use_cache=false)
 		return add(target,use_cache)
@@ -209,207 +186,6 @@ class WpTracker < Wmap::SiteTracker
 	#	return Hash.new
 	end
 
-  # Wordpress detection checkpoint - readme.html
-  def wp_readme?(url)
-		site = url_2_site(url)
-    readme_url=site + "readme.html"
-    k=Wmap::UrlChecker.new
-    if k.response_code(readme_url) == 200
-      k=nil
-      doc=open_page(readme_url)
-      title=doc.css('title')
-      if title.to_s =~ /wordpress/i
-        return true
-      else
-        return false
-      end
-    else
-      k=nil
-      return false
-    end
-	rescue => ee
-		puts "Exception on method #{__method__} for site #{url}: #{ee}" if @verbose
-		return false
-  end
-
-  # Wordpress detection checkpoint - install.css
-  def wp_css?(url)
-		site = url_2_site(url)
-    css_url = site + "wp-admin/css/install.css"
-    k=Wmap::UrlChecker.new
-    if k.response_code(css_url) == 200
-      k=nil
-      parser = CssParser::Parser.new
-      parser.load_uri!(css_url)
-      rule = parser.find_by_selector('#logo a')
-      if rule.length >0
-        if rule[0] =~ /wordpress/i
-          return true
-        end
-      end
-    else
-      k=nil
-      return false
-    end
-    return false
-	rescue => ee
-		puts "Exception on method #{__method__} for site #{url}: #{ee}" if @verbose
-		return false
-  end
-
-  # Wordpress detection checkpoint - meta generator
-  def wp_meta?(url)
-		site=url_2_site(url)
-    k=Wmap::UrlChecker.new
-    if k.response_code(site) == 200
-      k=nil
-      doc=open_page(site)
-      meta=doc.css('meta')
-      if meta.to_s =~ /wordpress/i
-        return true
-      else
-        return false
-      end
-    end
-		return false
-	rescue => ee
-		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
-		return false
-  end
-
-	# Wordpress detection checkpoint - wp-login
-  def wp_login?(url)
-		site=url_2_site(url)
-		login_url=site + "wp-login.php"
-    k=Wmap::UrlChecker.new
-    if k.response_code(login_url) == 200
-      k=nil
-      doc=open_page(login_url)
-      links=doc.css('link')
-      if links.to_s =~ /login.min.css/i
-        return true
-      else
-        return false
-      end
-    end
-		return false
-	rescue => ee
-		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
-		return false
-  end
-
-	# Wordpress detection checkpoint - xml-rpc
-  def wp_rpc?(url)
-		site=url_2_site(url)
-		rpc_url=site + "xmlrpc.php"
-    k=Wmap::UrlChecker.new
-		#puts "res code", k.response_code(rpc_url)
-    if k.response_code(rpc_url) == 405 # method not allowed
-      k=nil
-      return true
-    end
-		return false
-	rescue => ee
-		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
-		return false
-  end
-
-	# Extract the WordPress version
-	def wp_ver(url)
-		if !wp_ver_readme(url).nil?
-			puts "WordPress version found by wp_ver_readme method. " if @verbose
-			return wp_ver_readme(url)
-		elsif !wp_ver_login(url,"login.min.css").nil?
-			puts "WordPress version found by login.min.css file. " if @verbose
-			return wp_ver_login(url,"login.min.css")
-		elsif !wp_ver_login(url,"buttons.min.css").nil?
-			puts "WordPress version found by buttons.min.css file. " if @verbose
-			return wp_ver_login(url,"buttons.min.css")
-		elsif !wp_ver_login(url,"wp-admin.min.css").nil?
-			puts "WordPress version found by wp-admin.min.css file. " if @verbose
-			return wp_ver_login(url,"wp-admin.min.css")
-		elsif !wp_ver_meta(url).nil?
-			puts "WordPress version found by wp_ver_meta method. " if @verbose
-			return wp_ver_meta(url)
-		else
-			return nil
-		end
-	rescue => ee
-		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
-		return nil
-	end
-
-	# Identify wordpress version through the login page
-  def wp_ver_login(url,pattern)
-		puts "Check for #{pattern}" if @verbose
-		site=url_2_site(url)
-		login_url=site + "wp-login.php"
-    k=Wmap::UrlChecker.new
-		#puts "Res code: #{k.response_code(login_url)}" if @verbose
-    if k.response_code(login_url) == 200
-      doc=open_page(login_url)
-			#puts doc.inspect
-      links=doc.css('link')
-			#puts links.inspect if @verbose
-			links.each do |tag|
-	      if tag.to_s.include?(pattern)
-					puts tag.to_s if @verbose
-					k=nil
-	        return tag.to_s.scan(/[\d+\.]+\d+/).first
-	      end
-			end
-    end
-    k=nil
-    return nil
-	rescue => ee
-		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
-		return nil
-  end
-
-	# Identify wordpress version through the meta link
-  def wp_ver_meta(url)
-		site=url_2_site(url)
-    k=Wmap::UrlChecker.new
-    if k.response_code(site) == 200
-      doc=open_page(site)
-			#puts doc.inspect
-      meta=doc.css('meta')
-			#puts meta.inspect
-			meta.each do |tag|
-	      if tag['content'].to_s =~ /wordpress/i
-					#puts tag.to_s
-					k=nil
-	        return tag['content'].to_s.scan(/[\d+\.]+\d+/).first
-	      end
-			end
-    end
-    k=nil
-    return nil
-	rescue => ee
-		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
-		return nil
-  end
-
-	# Wordpress version detection via - readme.html
-  def wp_ver_readme(url)
-		site=url_2_site(url)
-    readme_url=site + "readme.html"
-    k=Wmap::UrlChecker.new
-		puts "Res code: #{k.response_code(readme_url)}" if @verbose
-    if k.response_code(readme_url) == 200
-      k=nil
-      doc=open_page(readme_url)
-			puts doc if @verbose
-      logo=doc.css('h1#logo')[0]
-      puts logo.inspect if @verbose
-			return logo.to_s.scan(/[\d+\.]+\d+/).first
-    end
-    k=nil
-    return nil
-	rescue => ee
-		puts "Exception on method #{__method__} for url #{url}: #{ee}" if @verbose
-		return nil
-	end
 
 end
 end