wmap 2.7.7 → 2.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 559cac84dd38902d968cc9e7327e77115ae3f946020caa21b7743ceb5777a96f
-  data.tar.gz: a4dcc0eafc10d7497c47c1f9955774b880ae5cdafa5ed0c6904ef5362087bd98
+  metadata.gz: a6632168c88c35189b601d225ebbf99fddd034a561814e5fe34b2d57bb79c75c
+  data.tar.gz: 4beab7d92e6a5e4258d37dfa52a64f4edaf06d8c4213331cc9833d0be5cc70aa
 SHA512:
-  metadata.gz: 0b430ed1da47cefd8cb8a7bedddd75ed2e7b1dafbfe94cdb2185ad2eb7e26d69a1429e79a9ece6b3cc68d6d964161f092e1143a00c0c64e241e930304d9e5a65
-  data.tar.gz: fe4d50b292849e51c202f47083c0837228d11934eb0d71c5a6262da24467480ce26994f5db91bdae5cf3b52dc0e6ef4fef698396575e820779c7420c731d0d87
+  metadata.gz: 46482f94126bc1ad4af322cb23894e77ee5a0f85dc31741e25b75752be4a47b1a620c3d1a6f28786a2f53e37eecde42b571f011e770e4bacdaf00012d737ea9e
+  data.tar.gz: b5ed0e1af39d8b7910b0d77b5bde99461293a7e985d5e587405bbf929cae8bffa5a32a1cf31eb142f823a11057377537adecf9ee6fb7f475fb2252e9d3edc2b1

data/bin/wmap

@@ -13,7 +13,7 @@ parser = OptionParser.new do|opts|
 	opts.on('-d', '--data_dir data_dir', 'Web Mapper local cache data directory') do |data_dir|
 		options[:data_dir] = data_dir;
 	end
-	opts.on('-t', '--target target', 'Web Mapper target') do |target|
+	opts.on('-t', '--target target', 'Web Mapper target / seed for discovery') do |target|
 		options[:target] = target;
 	end
 	opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
@@ -29,6 +29,10 @@ parser.parse!
 # print program banner
 puts Wmap.banner
 # print_usage unless options[:target]
+unless options[:target]
+	puts "Usage:   $ wmap -h"
+	exit 1
+end
 
 # Preparing - check out the working logs directory
 if options[:data_dir]
@@ -129,6 +133,10 @@ Wmap.wlog(dis_urls.keys, "wmap", Log_dir+"discovered_urls.log") unless dis_urls.
 Wmap.wlog(dis_sites.keys, "wmap", Log_dir+"discovered_sites.log") unless dis_sites.empty?
 #crawler.wlog(c_start.keys,Log_dir+"crawler.log")
 #crawler.wlog(c_done.keys,Log_dir+"crawler.log")
+
+
+# Save the current disovery urls only to a specific file, patched 07/23/2021
+crawler.save_discovered_urls(Log_dir+"cur_urls.log")
 crawler=nil
 
 
@@ -138,22 +146,23 @@ when nil,[]
 	puts "No new site found. There is no change to the site tracking data repository. "
 else
 	puts "Automatically save the discovery results into the site tracking data repository: "
+	inventory = Wmap::SiteTracker.instance
 	if options[:target] && options[:data_dir]
 		puts "Start the SiteTracker with the optional directory setter. "
-		inventory=Wmap::SiteTracker.instance
 		inventory.data_dir = options[:data_dir]
+		#inventory.verbose = true
 		inventory.sites_file = inventory.data_dir + "/" + "sites"
 		inventory.load_site_stores_from_file(inventory.sites_file)
 	elsif options[:target]
 		puts "Start the SiteTracker. "
-		inventory=Wmap::SiteTracker.instance
 	else
 		abort "Error firing up SiteTracker instance!"
 	end
 	new_sites=inventory.adds(dis_sites.keys-["",nil])
-	if new_sites.size>0 && options[:data_dir]
+	puts "Newly discovery sties: #{new_sites}"
+	if options[:data_dir]
 		inventory.save!(inventory.sites_file)
-	elsif new_sites.size>0
+	else
 		inventory.save!
 	end
 	inventory=nil
@@ -162,17 +171,17 @@ end
 
 
 # seventh step - update the hosts repository
+puts "Invoke the HostTracker with optional directory setter."
+host_tracker = Wmap::HostTracker.instance
 if options[:target] && options[:data_dir]
-	puts "Invoke the HostTracker with optional directory setter."
-	host_tracker = Wmap::HostTracker.instance
+	puts puts "Invoke the HostTracker with options: #{options[:data_dir]}, #{options[:target]}"
 	host_tracker.verbose=options[:verbose]
 	host_tracker.data_dir = options[:data_dir]
 	host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
 	host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
 elsif options[:target]
-	puts puts "Invoke the HostTracker."
-	host_tracker = Wmap::HostTracker.instance
-	host_tracker.verbose=options[:verbose]
+	puts puts "Invoke the HostTracker with option: #{options[:target]}."
+	#host_tracker.verbose=options[:verbose]
 else
 	abort "Error firing up HostTracker instance!"
 end

data/dicts/tlds.txt

@@ -1,5 +1,5 @@
 # http://data.iana.org/TLD/tlds-alpha-by-domain.txt
-# Version 2018110500, Last Updated Mon Nov  5 07:07:02 2018 UTC
+# Version 2020033000, Last Updated Mon Mar 30 07:07:01 2020 UTC
 AAA
 AARP
 ABARTH
@@ -16,7 +16,6 @@ ACCENTURE
 ACCOUNTANT
 ACCOUNTANTS
 ACO
-ACTIVE
 ACTOR
 AD
 ADAC
@@ -146,7 +145,6 @@ BIZ
 BJ
 BLACK
 BLACKFRIDAY
-BLANCO
 BLOCKBUSTER
 BLOG
 BLOOMBERG
@@ -155,7 +153,6 @@ BM
 BMS
 BMW
 BN
-BNL
 BNPPARIBAS
 BO
 BOATS
@@ -214,7 +211,6 @@ CARE
 CAREER
 CAREERS
 CARS
-CARTIER
 CASA
 CASE
 CASEIH
@@ -247,7 +243,6 @@ CHEAP
 CHINTAI
 CHRISTMAS
 CHROME
-CHRYSLER
 CHURCH
 CI
 CIPRIANI
@@ -299,6 +294,7 @@ COUNTRY
 COUPON
 COUPONS
 COURSES
+CPA
 CR
 CREDIT
 CREDITCARD
@@ -360,9 +356,7 @@ DNP
 DO
 DOCS
 DOCTOR
-DODGE
 DOG
-DOHA
 DOMAINS
 DOT
 DOWNLOAD
@@ -371,7 +365,6 @@ DTV
 DUBAI
 DUCK
 DUNLOP
-DUNS
 DUPONT
 DURBAN
 DVAG
@@ -392,7 +385,6 @@ ENERGY
 ENGINEER
 ENGINEERING
 ENTERPRISES
-EPOST
 EPSON
 EQUIPMENT
 ER
@@ -408,7 +400,6 @@ EU
 EUROVISION
 EUS
 EVENTS
-EVERBANK
 EXCHANGE
 EXPERT
 EXPOSED
@@ -488,6 +479,7 @@ GAME
 GAMES
 GAP
 GARDEN
+GAY
 GB
 GBIZ
 GD
@@ -580,7 +572,6 @@ HOMEGOODS
 HOMES
 HOMESENSE
 HONDA
-HONEYWELL
 HORSE
 HOSPITAL
 HOST
@@ -634,7 +625,6 @@ IQ
 IR
 IRISH
 IS
-ISELECT
 ISMAILI
 IST
 ISTANBUL
@@ -699,12 +689,10 @@ KYOTO
 KZ
 LA
 LACAIXA
-LADBROKES
 LAMBORGHINI
 LAMER
 LANCASTER
 LANCIA
-LANCOME
 LAND
 LANDROVER
 LANXESS
@@ -725,7 +713,6 @@ LEGO
 LEXUS
 LGBT
 LI
-LIAISON
 LIDL
 LIFE
 LIFEINSURANCE
@@ -744,6 +731,7 @@ LIVING
 LIXIL
 LK
 LLC
+LLP
 LOAN
 LOANS
 LOCKER
@@ -819,7 +807,6 @@ MN
 MO
 MOBI
 MOBILE
-MOBILY
 MODA
 MOE
 MOI
@@ -827,7 +814,6 @@ MOM
 MONASH
 MONEY
 MONSTER
-MOPAR
 MORMON
 MORTGAGE
 MOSCOW
@@ -835,7 +821,6 @@ MOTO
 MOTORCYCLES
 MOV
 MOVIE
-MOVISTAR
 MP
 MQ
 MR
@@ -854,7 +839,6 @@ MY
 MZ
 NA
 NAB
-NADEX
 NAGOYA
 NAME
 NATIONWIDE
@@ -955,7 +939,6 @@ PHOTO
 PHOTOGRAPHY
 PHOTOS
 PHYSIO
-PIAGET
 PICS
 PICTET
 PICTURES
@@ -1152,18 +1135,16 @@ SONG
 SONY
 SOY
 SPACE
-SPIEGEL
 SPORT
 SPOT
 SPREADBETTING
 SR
 SRL
-SRT
+SS
 ST
 STADA
 STAPLES
 STAR
-STARHUB
 STATEBANK
 STATEFARM
 STC
@@ -1211,7 +1192,6 @@ TEAM
 TECH
 TECHNOLOGY
 TEL
-TELEFONICA
 TEMASEK
 TENNIS
 TEVA
@@ -1271,7 +1251,6 @@ TZ
 UA
 UBANK
 UBS
-UCONNECT
 UG
 UK
 UNICOM
@@ -1305,7 +1284,6 @@ VIP
 VIRGIN
 VISA
 VISION
-VISTAPRINT
 VIVA
 VIVO
 VLAANDEREN
@@ -1324,7 +1302,6 @@ WALMART
 WALTER
 WANG
 WANGGOU
-WARMAN
 WATCH
 WATCHES
 WEATHER
@@ -1452,13 +1429,14 @@ XN--MGBA7C0BBN0A
 XN--MGBAAKC7DVF
 XN--MGBAAM7A8H
 XN--MGBAB2BD
+XN--MGBAH1A3HJKRD
 XN--MGBAI9AZGQP6J
 XN--MGBAYH7GPA
-XN--MGBB9FBPOB
 XN--MGBBH1A
 XN--MGBBH1A71E
 XN--MGBC0A9AZCG
 XN--MGBCA7DZDO
+XN--MGBCPQ6GPA1A
 XN--MGBERP4A5D4AR
 XN--MGBGU82A
 XN--MGBI4ECEXP
@@ -1484,8 +1462,10 @@ XN--P1AI
 XN--PBT977C
 XN--PGBS0DH
 XN--PSSY2U
+XN--Q7CE6A
 XN--Q9JYB4C
 XN--QCKA1PMC
+XN--QXA6A
 XN--QXAM
 XN--RHQV96G
 XN--ROVU88B
@@ -1530,7 +1510,6 @@ ZAPPOS
 ZARA
 ZERO
 ZIP
-ZIPPO
 ZM
 ZONE
 ZUERICH

data/lib/wmap/cidr_tracker.rb

@@ -18,7 +18,7 @@ class Wmap::CidrTracker
 		@verbose=params.fetch(:verbose, false)
 		@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
 		Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
-		@cidr_seeds=params.fetch(:cidr_seeds, @data_dir + 'cidrs')
+		@cidr_seeds=params.fetch(:cidr_seeds, @data_dir + '/' + 'cidrs')
 		File.write(@cidr_seeds, "") unless File.exist?(@cidr_seeds)
 		load_cidr_blks_from_file(@cidr_seeds)
 	end
@@ -99,8 +99,8 @@ class Wmap::CidrTracker
 		#@known_cidr_blks_asce_index=NetAddr.sort(@known_cidr_blks.keys, :Desc=>false)
 		@known_cidr_blks_asce_index=@known_cidr_blks.keys.sort
 		@known_cidr_blks_desc_index=@known_cidr_blks_asce_index.reverse
-	#rescue => ee
-	#	puts "Exception on method #{__method__}: #{ee}" # if @verbose
+	rescue => ee
+		puts "Exception on method #{__method__}: #{ee}" # if @verbose
 	end
 
 	# 'setter' to remove an entry to CIDR store @known_cidr_blks
@@ -167,6 +167,7 @@ class Wmap::CidrTracker
 			known = cidr4.contains?(ip+'/32')
 			break if known
 		end
+		return known
 	rescue => ee
 		puts "Exception on method #{__method__}: #{ee}" if @verbose
 		return false

data/lib/wmap/domain_tracker.rb

@@ -169,7 +169,7 @@ class Wmap::DomainTracker
 				end
 			end
 			@known_internet_domains.merge!(results)
-			puts "Done loading entries."
+			puts "Done loading domain entries."
 			return results
 		else
 			puts "Error: no entry is loaded. Please check your list and try again."
@@ -262,7 +262,7 @@ class Wmap::DomainTracker
 		when "Wmap::DomainTracker::SubDomain"
 			return @known_internet_sub_domains.key?(domain)
 		else
-			return nil
+			return false
 		end
 	rescue => ee
 		puts "Exception on method #{__method__}: #{ee}" if @verbose

data/lib/wmap/domain_tracker/sub_domain.rb

@@ -70,7 +70,7 @@ class SubDomain < Wmap::DomainTracker
 				end
 			end
 			@known_internet_sub_domains.merge!(results)
-			puts "Done loading entries."
+			puts "Done loading sub_domain entries."
 			return results
 		else
 			puts "Error: no entry is loaded. Please check your list and try again."

data/lib/wmap/host_tracker.rb

@@ -27,13 +27,13 @@ class Wmap::HostTracker
 		@max_parallel=params.fetch(:max_parallel, 40)
 		# Initialize the instance variables
 		File.write(@hosts_file, "") unless File.exist?(@hosts_file)
-		load_known_hosts_from_file(@hosts_file)
+		@known_hosts=load_known_hosts_from_file(@hosts_file)
 	end
 
 	# Setter to load the known hosts from the local hosts file into a class instance
 	def load_known_hosts_from_file (f_hosts=@hosts_file)
 		puts "Loading local hosts from file: #{f_hosts} ..." if @verbose
-		@known_hosts=Hash.new
+		known_hosts=Hash.new
 		@alias = Hash.new
 		File.write(f_hosts, "") unless File.exist?(f_hosts)
 		f=File.open(f_hosts, 'r')
@@ -43,11 +43,11 @@ class Wmap::HostTracker
 			key=entry[0].downcase
 			value=entry[1]
 			puts "Loading key value pair: #{key} - #{value}" if @verbose
-			@known_hosts[key] = Hash.new unless @known_hosts.key?(key)
-			@known_hosts[key]= value
+			known_hosts[key] = Hash.new unless known_hosts.key?(key)
+			known_hosts[key]= value
 			# For reverse host lookup
-			@known_hosts[value] = Hash.new unless @known_hosts.key?(value)
-			@known_hosts[value] = key
+			known_hosts[value] = Hash.new unless known_hosts.key?(value)
+			known_hosts[value] = key
 			# Count the number of alias for the recorded IP
 			if @alias.key?(value)
 				@alias[value]+=1
@@ -56,11 +56,12 @@ class Wmap::HostTracker
 			end
 		end
 		f.close
-		return @known_hosts
-	rescue => ee
-		puts "Exception on method #{__method__}: #{ee}"
 		return known_hosts
+		#rescue => ee
+		#	puts "Exception on method #{__method__}: #{ee}"
+		#	return known_hosts
 	end
+	alias_method :load, :load_known_hosts_from_file
 
 	# Save the current local hosts hash table into a (random) data repository file
 	def save_known_hosts_to_file!(f_hosts=@hosts_file)
@@ -96,30 +97,42 @@ class Wmap::HostTracker
 		puts "Exception on method #{__method__}: #{ee}"
 	end
 
+  # determine if host is part of trusted (known) root domains
+	def is_trusted?(host)
+		puts "Determin if host #{host} is part of trusted root domains" if @verbose
+		root=get_domain_root(host)
+		puts "Domain root: #{root}" if @verbose
+		domain_tracker=Wmap::DomainTracker.instance
+		domain_tracker.data_dir=@data_dir
+		domain_tracker.domains_file = domain_tracker.data_dir + "/" + "domains"
+		domain_tracker.load_domains_from_file
+		if domain_tracker.domain_known?(root)
+			domain_tracker=nil
+			return true
+		else
+			domain_tracker=nil
+			return false
+		end
+	end
+
 	# Setter to add host entry to the cache once at a time
 	def add(host)
 		puts "Add entry to the local host repository: #{host}"
 		host=host.strip.downcase unless host.nil?
+		root=get_domain_root(host)
 		unless @known_hosts.key?(host)
 			ip=host_2_ip(host)
 			record=Hash.new
 			if is_ip?(ip)
 				# filter host to known domains only
-				root=get_domain_root(host)
-				puts "Domain root: #{root}" if @verbose
-				domain_tracker=Wmap::DomainTracker.instance
-				domain_tracker.data_dir=@data_dir
-				domain_tracker.domains_file = domain_tracker.data_dir + "domains"
-				domain_tracker.load_domains_from_file
-				if domain_tracker.domain_known?(root)
-					domain_tracker=nil
+				if is_trusted?(host)
 					record[host]=ip
 					record[ip]=host
 					puts "Host data repository entry loaded: #{host} <=> #{ip}"
 					# Replace instance with the class variable to avoid potential race condition under parallel engine
 					# add additional logic to update the sub-domain table as well, 02/10/2014
 					sub=get_sub_domain(host)
-					if sub!=root
+					if sub!=nil
 						tracker=Wmap::DomainTracker::SubDomain.instance
 						tracker.data_dir=@data_dir
 						tracker.sub_domains_file = tracker.data_dir + "sub_domains"
@@ -142,8 +155,8 @@ class Wmap::HostTracker
 		else
 			puts "Host is already exist. Skip: #{host}"
 		end
-	rescue => ee
-		puts "Exception on method #{__method__}: #{ee}" if @verbose
+	#rescue => ee
+	#	puts "Exception on method #{__method__}: #{ee}" if @verbose
 	end
 
 	# Setter to add host entry to the local hosts in batch (from an array)
@@ -164,7 +177,7 @@ class Wmap::HostTracker
 				end
 			end
 			@known_hosts.merge!(results)
-			puts "Done loading entries."
+			puts "Done loading host entries."
 			return results
 		else
 			puts "Error: empty list - no entry is loaded. Please check your input list and try again."

data/lib/wmap/site_tracker.rb

@@ -77,7 +77,9 @@ class Wmap::SiteTracker
 		f.write "# Local site store created by class #{self.class} method #{__method__} at: #{timestamp}\n"
 		f.write "# Website,Primary IP,Port,Hosting Status,Server,Response Code,MD5 Finger-print,Redirection,Timestamp\n"
 		@known_sites.keys.sort.map do |key|
-			f.write "#{key},#{@known_sites[key]['ip']},#{@known_sites[key]['port']},#{@known_sites[key]['status']},#{@known_sites[key]['server']},#{@known_sites[key]['code']},#{@known_sites[key]['md5']},#{@known_sites[key]['redirection']},#{@known_sites[key]['timestamp']}\n"
+			if is_trusted?(key)
+				f.write "#{key},#{@known_sites[key]['ip']},#{@known_sites[key]['port']},#{@known_sites[key]['status']},#{@known_sites[key]['server']},#{@known_sites[key]['code']},#{@known_sites[key]['md5']},#{@known_sites[key]['redirection']},#{@known_sites[key]['timestamp']}\n"
+			end
 		end
 		f.close
 		puts "site store table is successfully saved: #{file_sites}"
@@ -94,6 +96,24 @@ class Wmap::SiteTracker
 		puts "Exception on method #{__method__}: #{ee}"
 	end
 
+	# determine site is trusted based on the known domains
+	def is_trusted?(site)
+		trusted=false
+		host=url_2_host(site)
+		root=get_domain_root(host)
+		domain_tracker=Wmap::DomainTracker.instance
+		domain_tracker.data_dir=@data_dir
+		domain_tracker.domains_file=@data_dir + "/" + "domains"
+		File.write(domain_tracker.domains_file, "") unless File.exist?(domain_tracker.domains_file)
+		domain_tracker.load_domains_from_file(domain_tracker.domains_file)
+		trusted=domain_tracker.domain_known?(root)
+		domain_tracker=nil
+		return trusted
+	rescue => ee
+		puts "Exception on method #{__method__}: #{ee}"
+		return trusted
+	end
+
 	# Setter to add site entry to the cache one at a time
 	def add(site)
 		puts "Add entry to the site store: #{site}"
@@ -132,6 +152,10 @@ class Wmap::SiteTracker
 			end
 		end
 		# add record only if trusted
+		host_tracker = Wmap::HostTracker.instance
+		host_tracker.data_dir= @data_dir
+		host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
+		host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
 		if trusted
 			# Add logic to check site status before adding it
 			checker=Wmap::UrlChecker.new(:data_dir=>@data_dir).check(site)
@@ -144,10 +168,6 @@ class Wmap::SiteTracker
 				raise "Site is currently down. Skip #{site}" if checker['code']==10000
 			end
 			raise "Exception on add method - Fail to resolve the host-name: Host - #{host}, IP - #{ip}. Skip #{site}" unless is_ip?(ip)
-			host_tracker = Wmap::HostTracker.instance
-			host_tracker.data_dir= @data_dir
-			host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
-			host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
 			# Update the local host table when necessary
 			if is_ip?(host)
 				# Case #1: Trusted site contains IP
@@ -341,8 +361,8 @@ class Wmap::SiteTracker
 		else
 			puts "Error: no entry is loaded. Please check your list and try again."
 		end
-	#rescue => ee
-	#	puts "Exception on method #{__method__}: #{ee}" if @verbose
+	rescue => ee
+		puts "Exception on method #{__method__}: #{ee}" if @verbose
 	end
 	alias_method :dels, :bulk_delete
 

data/lib/wmap/site_tracker/wp_tracker.rb

@@ -174,7 +174,7 @@ class WpTracker < Wmap::SiteTracker
 				end
 			end
 			@known_wp_sites.merge!(results)
-			puts "Done loading entries."
+			puts "Done loading wp entries."
 			return results
 		else
 			puts "Error: no entry is loaded. Please check your list and try again."

data/lib/wmap/url_checker.rb

@@ -88,9 +88,9 @@ class Wmap::UrlChecker
 		checker['redirection']=nil
 		checker['timestamp']=timestamp
 		return checker
-	rescue Exception => ee
-		puts "Exception on method #{__method__} for #{url}: #{ee}" # if @verbose
-		return nil
+	#rescue Exception => ee
+	#	puts "Exception on method #{__method__} for #{url}: #{ee}" # if @verbose
+	#	return nil
 	end
 	alias_method :check, :url_worker
 

data/lib/wmap/url_crawler.rb

@@ -17,7 +17,8 @@ require "parallel"
 class Wmap::UrlCrawler
 	include Wmap::Utils
 
-	attr_accessor :http_timeout, :crawl_page_limit, :crawl_depth, :max_parallel, :verbose, :data_dir
+	attr_accessor :http_timeout, :crawl_page_limit, :crawl_depth, :max_parallel, \
+				:verbose, :data_dir, :user_agent
 	attr_reader :discovered_urls_by_crawler, :visited_urls_by_crawler, :crawl_start, :crawl_done
 	# Global variable used to store the combined result of all the forked child processes. Note that class variable
 	# would not be able to pass the result due the limitation of IO Pipe communication mechanism used by 'parallel' fork manager
@@ -35,13 +36,16 @@ class Wmap::UrlCrawler
 		@crawl_depth=params.fetch(:crawl_depth, 4)
 		@crawl_page_limit=params.fetch(:crawl_page_limit, 1000)
 		@max_parallel=params.fetch(:max_parallel, 40)
+		@user_agent=params.fetch(:user_agent, "OWASP WMAP Spider")
 		# Discovered data store
 		@discovered_urls_by_crawler=Hash.new
 		@visited_urls_by_crawler=Hash.new
 		@crawl_start=Hash.new
 		@crawl_done=Hash.new
 		Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
-		@log_file=@data_dir + "/../logs/crawler.log"
+		@log_dir=@data_dir + "/logs/"
+		Dir.mkdir(@log_dir) unless Dir.exist?(@log_dir)
+		@log_file=@log_dir + "crawler.log"
 	end
 
 	# Pre-crawl profiler, to be used for network profiling to maximum the crawler performance.
@@ -86,7 +90,7 @@ class Wmap::UrlCrawler
 
     # The worker instance of crawler who perform the labour work
 	def crawl_worker(url0)
-		puts "Please be aware that it may take a while to crawl #{url0}, depending on the site's responsiveness and the amount of contents."
+		puts "Please be aware that it may take a while to crawl #{url0}, depending on the site's responsiveness and discovery contents."
 		# Input URL sanity check first
 		if is_url?(url0)
 			host=url_2_host(url0)
@@ -216,14 +220,14 @@ class Wmap::UrlCrawler
 	alias_method :crawl_file, :crawl_workers_on_file
 
   # Wrapper for the OpenURI open method - create an open_uri object and return the reference upon success
-	def open_url(url)
+	def open_url(url,user_agent=@user_agent)
 		puts "Open url #{url} by creating an open_uri object. Return the reference upon success." if @verbose
 		if url =~ /http\:/i
 			# patch for allow the 'un-safe' URL redirection i.e. https://www.example.com -> http://www.example.com
-			url_object = open(url, :allow_redirections=>:safe, :read_timeout=>Max_http_timeout/1000)
+			url_object = open(url, :allow_redirections=>:safe, :read_timeout=>Max_http_timeout/1000, "User-Agent"=>user_agent)
 			#url_object = open(url)
 		elsif url =~ /https\:/i
-			url_object = open(url,:ssl_verify_mode => 0, :allow_redirections =>:safe, :read_timeout=>Max_http_timeout/1000)
+			url_object = open(url, :ssl_verify_mode=>0, :allow_redirections=>:safe, :read_timeout=>Max_http_timeout/1000, "User-Agent"=>user_agent)
 			#url_object = open(url,:ssl_verify_mode => 0)
 		else
 			raise "Invalid URL format - please specify the protocol prefix http(s) in the URL: #{url}"
@@ -258,22 +262,6 @@ class Wmap::UrlCrawler
     return nil
   end
 
-=begin
-    # Wrapper for the Nokogiri DOM parser
-	def parse_html(html_body)
-		begin
-			#puts "Parsing the html content: #{html_body}. Return DOM " if @verbose
-      doc = Nokogiri::HTML(html_body)
-			#puts "Successfully crawling the url: #{url_object.base_uri.to_s}" if @verbose
-			#puts "doc: #{doc}" if @verbose
-			return doc
-    rescue => ee
-      puts "Exception on method #{__method__}: #{ee}" if @verbose
-      return nil
-    end
-	end
-=end
-
   # Search 'current_url' and return found URLs under the same domain
 	def find_urls_on_page(doc, current_url)
 		puts "Search and return URLs within the doc: #{doc}" if @verbose

data/lib/wmap/url_crawler/adware_tag.rb

@@ -119,7 +119,7 @@ module Wmap
 					end
 				end
 				@tag_store.merge!(results)
-				puts "Done loading entries."
+				puts "Done loading adware entries."
         tags = nil
 				return results
 			else

data/lib/wmap/utils/domain_root.rb

@@ -192,7 +192,7 @@ module Wmap
 	# Function to print instance variable - General top level domain list
 	def print_gtld
 		puts @gtld
-    return @gtld 
+    return @gtld
 	end
 
 	# Function to print instance variable - Country code top-level domain list

data/lib/wmap/utils/logger.rb

@@ -8,46 +8,43 @@
 
 
 module Wmap
- module Utils	
+ module Utils
   # Module to log debugging and other messages
-  module Logger	
+  module Logger
 	extend self
 	# Append information into the log file for the trouble-shooting purpose
 	def wlog (obj, agent, file)
 		puts "Writing #{obj} into log file: #{file}" if @verbose
-		begin
-			return false if obj.nil? 
-			# 01/27/2015, implementing singleton pattern for the logger
-			@@f=File.open(file,'a')	
-			timestamp=Time.now
-			case obj
-			when Array
-				if obj.size >= 0
-					@@f.write "#{timestamp}: #{agent}: \n"
-					obj.map { |x| @@f.write "  #{x}\n" }
-					puts "The list is successfully saved into the log file: #{file} " if @verbose
-				end
-			when Hash
-				if obj.length >= 0
-					@@f.write "#{timestamp}: #{agent}: \n"
-					obj.each_value { |value| @@f.write "  #{value}\n" }
-					puts "The hash is successfully saved into the log file: #{file} " if @verbose
-				end
-			when String
-				@@f.write "#{timestamp}: #{agent}: #{obj}\n"
-				puts "The string is successfully saved into the log file: #{file} " if @verbose
-			else
-				#do nothing
-				puts "Un-handled exception on: #{obj}" if @verbose
+		return false if obj.nil?
+		@@f=File.open(file,'a')
+		timestamp=Time.now
+		case obj
+		when Array
+			if obj.size >= 0
+				@@f.write "#{timestamp}: #{agent}: \n"
+				obj.map { |x| @@f.write "  #{x}\n" }
+				puts "The list is successfully saved into the log file: #{file} " if @verbose
 			end
-			@@f.close
-			return true
-		rescue => ee
-			puts "Exception on method #{__method__}: #{ee}" if @verbose
-			return false
-		end	
+		when Hash
+			if obj.length >= 0
+				@@f.write "#{timestamp}: #{agent}: \n"
+				obj.each_value { |value| @@f.write "  #{value}\n" }
+				puts "The hash is successfully saved into the log file: #{file} " if @verbose
+			end
+		when String
+			@@f.write "#{timestamp}: #{agent}: #{obj}\n"
+			puts "The string is successfully saved into the log file: #{file} " if @verbose
+		else
+			#do nothing
+			puts "Un-handled exception on: #{obj}" if @verbose
+		end
+		@@f.close
+		return true
+	rescue => ee
+		puts "Exception on method #{__method__}: #{ee}" if @verbose
+		return false
 	end
-	
-  end	
+
+  end
  end
 end

data/lib/wmap/utils/url_magic.rb

@@ -15,6 +15,7 @@ module Wmap
 
   # set hard stop limit of http time-out to 8 seconds, in order to avoid severe performance penalty for certain 'weird' site(s)
   Max_http_timeout=15000
+  User_agent = "OWASP WMAP Spider"
 
 	# Simple sanity check on a 'claimed' URL string.
 	def is_url?(url)
@@ -377,7 +378,8 @@ module Wmap
 
   # Given an URL, open the page, then return the DOM text from a normal user perspective
   def open_page(url)
-    args = {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE, allow_redirections: :safe, read_timeout: Max_http_timeout/1000}
+    args = {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE, allow_redirections: :safe, \
+      read_timeout: Max_http_timeout/1000, "User-Agent"=>User_agent}
     doc = Nokogiri::HTML(open(url, args))
     if doc.text.include?("Please enable JavaScript to view the page content")
       puts "Invoke headless chrome through webdriver ..." if @verbose
@@ -385,7 +387,7 @@ module Wmap
       #driver = Selenium::WebDriver.for :chrome
       # http://watir.com/guides/chrome/
       args = ['--ignore-certificate-errors', '--disable-popup-blocking', '--disable-translate', '--disk-cache-size 8192']
-      browser = Watir::Browser.new :chrome, headless: true, options: {args: args}
+      browser = Watir::Browser.new :chrome, headless: true, switches: %w[--user-agent=OWASP\ WMAP\ Spider]
       browser.goto(url)
       sleep(2) # wait for the loading
       doc = Nokogiri::HTML(browser.html)

data/lib/wmap/utils/utils.rb

@@ -159,20 +159,18 @@ module Wmap
 	# Simple test a host string format. Return true if it contains a valid internet domain sub-string. Note: Don't be confused with another method 'valid_dns_record?', which is a stricter and time-consuming test on the DNS server for a resolvable internet host.
 	def is_fqdn? (host)
 		puts "Validate the host-name format is valid: #{host}" if @verbose
-		begin
-			return false if is_ip?(host) or is_url?(host)
-			domain=get_domain_root(host)
-			if domain.nil?
-				return false
-			elsif is_domain_root?(domain)
-				return true
-			else
-				return false
-			end
-		rescue => ee
-			puts "Exception on method is_fqdn? for #{host}: #{ee}" if @verbose
+		return false if is_ip?(host) or is_url?(host)
+		domain=get_domain_root(host)
+		if domain.nil?
+			return false
+		elsif is_domain_root?(domain)
+			return true
+		else
 			return false
 		end
+#	rescue => ee
+#		puts "Exception on method is_fqdn? for #{host}: #{ee}" if @verbose
+#		return false
 	end
 	alias_method :is_host?, :is_fqdn?
 

data/lib/wmap/utils/wp_detect.rb

@@ -239,7 +239,11 @@ module Wmap
   	      if tag.to_s.include?(pattern)
   					puts tag.to_s if @verbose
   					k=nil
-  	        return tag.to_s.scan(/[\d+\.]+\d+/).first
+            if tag.to_s.scan(/[\d+\.]+\d+/).first =~ /\d+\./
+  	           return tag.to_s.scan(/[\d+\.]+\d+/).first
+             else
+               return nil
+             end
   	      end
   			end
       end

data/test/cidr_tracker_test.rb

@@ -0,0 +1,36 @@
+#--
+# Wmap
+#
+# A pure Ruby library for the Internet web application discovery and tracking.
+#
+# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
+#++
+# Unit Test File for Wmap::DomainTracker.instance class
+
+require "minitest/autorun"
+require "Wmap"
+
+class CidrTrackerTest < MiniTest::Unit::TestCase
+	include Wmap::Utils
+
+	def test_cidr_add
+		w = Wmap::CidrTracker.new
+		w.add("192.168.1.0/24")
+		assert_equal true, w.known_cidr_blks.key?("192.168.1.0/24")
+	end
+
+	def test_cidr_delete
+		w = Wmap::CidrTracker.new
+		w.add("10.0.0.0/8")
+		w.delete("10.0.0.0/8")
+		assert_equal false, w.known_cidr_blks.key?("10.0.0.0/8")
+	end
+
+	def test_ip_trusted?
+		w = Wmap::CidrTracker.new
+		w.add("192.168.1.0/24")
+		assert_equal true, w.ip_trusted?("192.168.1.1")
+		assert_equal true, w.ip_trusted?("192.168.1.255")
+	end
+
+end

data/test/utils_test.rb

@@ -12,7 +12,7 @@ require "Wmap"
 
 class UtilsTest < MiniTest::Unit::TestCase
 	include Wmap::Utils
-	
+
 	def test_sld_domain_conversion
 		assert_equal "yahoo.com", get_domain_root("yahoo.com")
 	end
@@ -28,75 +28,75 @@ class UtilsTest < MiniTest::Unit::TestCase
 	def test_is_domain_root_case_1?
 		assert_equal false, is_domain_root?("www.yahoo.co.uk")
 	end
-	
+
 	def test_is_domain_root_case_2?
 		assert_equal true, is_domain_root?("yahoo.co.uk")
 	end
-	
+
 	def test_get_sub_domain
 		assert_equal "mail.yahoo.co.uk", get_sub_domain("www.mail.yahoo.co.uk")
 	end
 
 	def test_is_url_case_1?
 		assert_equal true, is_url?("http://www.mail.yahoo.co.uk/")
-	end	
+	end
 
 	def test_is_url_case_2?
 		assert_equal true, is_url?("https://www.mail.yahoo.co.uk/")
-	end		
+	end
 
 	def test_is_url_case_3?
 		assert_equal false, is_url?("http://www.mail.yahoo.uii/")
-	end	
+	end
 
 	def test_is_url_case_4?
 		assert_equal false, is_url?("http:\\www.mail.yahoo.co.uk")
-	end	
-	
+	end
+
 	def test_is_ssl?
 		assert_equal false, is_ssl?("http://www.mail.yahoo.co.uk/")
-	end	
-	
+	end
+
 	def test_is_site?
 		assert_equal false, is_site?("https://login.yahoo.com/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
-	end	
-	
+	end
+
 	def test_url_2_host
 		assert_equal "login.yahoo.com", url_2_host("https://login.yahoo.com/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
-	end	
-	
+	end
+
 	def test_url_2_site_case_1
 		assert_equal "https://login.yahoo.com/", url_2_site("https://login.yahoo.com/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
-	end	
+	end
 
 	def test_url_2_site_case_2
 		assert_equal "https://login.yahoo.com/", url_2_site("https://login.yahoo.com?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
-	end	
+	end
 
 	def test_url_2_site_case_3
 		assert_equal "https://login.yahoo.com/", url_2_site("https://login.yahoo.com#.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
-	end	
-	
+	end
+
 	def test_url_2_path
 		assert_equal "/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com", url_2_path("https://login.yahoo.com/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com")
-	end	
+	end
 
 	def test_urls_on_same_domain?
 		assert_equal true, urls_on_same_domain?("https://login.yahoo.com/?.src=ym&.intl=us&.lang=en-US&.done=https%3a//mail.yahoo.com", "https://us-mg4.mail.yahoo.com/neo/launch?.rand=8hjd08hc6t1lq")
-	end	
+	end
 
 	def test_host_2_url_case_1
 		assert_equal "https://mail.yahoo.com/", host_2_url("mail.yahoo.com",443)
-	end	
+	end
 
 	def test_host_2_url_case_2
 		assert_equal "http://mail.yahoo.com/", host_2_url("mail.yahoo.com")
-	end	
-	
+	end
+
 	def test_make_absolute
 		assert_equal "http://games.yahoo.com/game/the-magic-snowman-flash.html", make_absolute("http://games.yahoo.com/","game/the-magic-snowman-flash.html")
 	end
-	
+
 	def test_create_absolute_url_from_base
 		assert_equal "http://images.search.yahoo.com/search/images?p=raiders", create_absolute_url_from_base("http://images.search.yahoo.com/images","/search/images?p=raiders")
 	end
@@ -108,7 +108,7 @@ class UtilsTest < MiniTest::Unit::TestCase
 	def test_normalize_url_case_1
 		assert_equal "http://images.search.yahoo.com/images/search/images?p=raiders", normalize_url("http://images.search.yahoo.com/./images/search/images?p=raiders")
 	end
-	
+
 	def test_normalize_url_case_2
 		assert_equal "http://images.search.yahoo.com/images/search/images?p=raiders", normalize_url("http://images.search.yahoo.com/../images/../search/images?p=raiders")
 	end
@@ -116,53 +116,58 @@ class UtilsTest < MiniTest::Unit::TestCase
 	def test_normalize_url_case_3
 		assert_equal "http://images.search.yahoo.com/images/search/images?p=raiders", normalize_url("http://images.search.yahoo.com./../images/../search/images?p=raiders")
 	end
-	
+
 	def test_is_ip_case_1?
 		assert_equal false, is_ip?("256.2.3.1")
-	end	
+	end
 
 	def test_is_ip_case_2?
 		assert_equal false, is_ip?("25.2.3.1.22")
-	end	
+	end
 
 	def test_is_ip_case_3?
 		assert_equal true, is_ip?("196.168.230.1")
-	end	
+	end
 
 	def test_is_fqdn_case_1?
 		assert_equal true, is_fqdn?("images.search.yahoo.com")
-	end	
+	end
 
 	def test_is_fqdn_case_2?
 		assert_equal true, is_fqdn?("yahoo.com")
-	end		
-	
+	end
+
 	def test_is_fqdn_case_3?
-		assert_equal false, is_fqdn?("images.search.yahoo")
-	end	
-	
+		# according to latest tlds list - http://data.iana.org/TLD/tlds-alpha-by-domain.txt
+		assert_equal true, is_fqdn?("images.search.yahoo")
+	end
+
 	def test_is_fqdn_case_4?
 		assert_equal false, is_fqdn?("images")
-	end	
-	
+	end
+
+	def test_is_fqdn_case_5?
+		assert_equal false, is_fqdn?("images.search.gargle")
+	end
+
 	def test_is_cidr_case_1?
 		assert_equal false, is_cidr?("196.168.230.1")
-	end	
+	end
 
 	def test_is_cidr_case_2?
 		assert_equal false, is_cidr?("196.168.2.257/12")
-	end	
-	
+	end
+
 	def test_is_cidr_case_3?
 		assert_equal true, is_cidr?("196.168.2.25/12")
-	end	
-	
+	end
+
 	def test_cidr_2_ips
 		assert_equal ["192.168.1.1"], cidr_2_ips("192.168.1.1/32")
-	end	
-	
+	end
+
 	def test_sort_ips
 		assert_equal ["192.168.1.1", "192.168.1.2", "192.168.2.1"], sort_ips(["192.168.1.2", "192.168.2.1","192.168.1.1"])
-	end		
-	
+	end
+
 end

data/version.txt

@@ -3,8 +3,8 @@
 ###############################################################################
 package = wmap
 # wmap version 2.0 == web_discovery version 1.5.3
-version = 2.7.7
-date = 2020-03-24
+version = 2.8.3
+date = 2021-07-26
 
 author = Sam (Yang) Li
 email = yang.li@owasp.org

data/wmap.gemspec

@@ -36,7 +36,7 @@ Gem::Specification.new do |s|
   s.description = "wmap is written to perform Internet web application / service discovery. The discovery results are designed to be automatically tracked by the software."
   s.email = info["email"]
   s.executables = ["wmap","wscan","wadd","wadds","wdel","wcheck","wdump","spiderBot","googleBot","updateAll","prime","deprime","refresh","trust","trusts","distrust","run_tests"]
-  s.files = ["CHANGELOG.md", "TODO", "settings/discovery_ports","data/","LICENSE.txt",
+  s.files = ["CHANGELOG.md", "TODO", "settings/discovery_ports", "LICENSE.txt",
 							"version.txt","README.md", "wmap.gemspec"]
   s.files += Dir['lib/*.rb'] + Dir['lib/wmap/*.rb'] + Dir['lib/wmap/**/*'] + Dir['bin/*'] + Dir['settings/*'] + Dir['demos/*'] + Dir['test/*'] + Dir['dicts/*']
   #s.homepage = "none"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wmap
 version: !ruby/object:Gem::Version
-  version: 2.7.7
+  version: 2.8.3
 platform: ruby
 authors:
 - Sam (Yang) Li
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-03-24 00:00:00.000000000 Z
+date: 2021-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dnsruby
@@ -234,7 +234,6 @@ files:
 - LICENSE.txt
 - README.md
 - TODO
-- bin/RHPG
 - bin/deprime
 - bin/distrust
 - bin/googleBot
@@ -251,7 +250,6 @@ files:
 - bin/wdel
 - bin/wdump
 - bin/wmap
-- bin/wmaps
 - bin/wscan
 - demos/bruter.rb
 - demos/dns_brutes.rb
@@ -308,7 +306,7 @@ files:
 - settings/discovery_ports
 - settings/google_keywords.txt
 - settings/google_locator.txt
-- settings/tag_signatures
+- test/cidr_tracker_test.rb
 - test/domain_tracker_test.rb
 - test/utils_test.rb
 - version.txt
@@ -335,9 +333,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: wmap
-rubygems_version: 2.7.10
-signing_key: 
+rubygems_version: 3.0.9
+signing_key:
 specification_version: 4
 summary: A pure Ruby web application and service discovery API.
 test_files: []

data/bin/RHPG

@@ -1,107 +0,0 @@
-#!/usr/bin/env ruby
-# Executable to lookup then merge site tech details into the RHPG asset spreadsheet in CSV format only
-#
-## Usage: RHPG [RHPG.csv]
-require "wmap"
-require "csv"
-include Wmap::Utils
-
-def print_usage
-	puts "Program to lookup then merge the site details into RHPG asset spreadsheet. \nUsage: RHPG [RHPG.csv]"
-end
-
-# Lookup the site store for a domain; then return the fingger print info of the site
-def site_tracker_lookup(domain)
-  tracker=Wmap::SiteTracker.instance
-	tracker.verbose=false
-  #first order search
-  tracker.known_sites.each do |key,val|
-    if key.include?(domain.strip.downcase) && key.include?("https")
-      tracker=nil
-      return [key] + val.values
-    end
-  end
-  #second order search
-  tracker.known_sites.each do |key,val|
-    if key.include?(domain.strip.downcase)
-      tracker=nil
-      return [key] + val.values
-    end
-  end
-  tracker=nil
-  return [nil]*9
-end
-
-# look up the wp site data store for a domain; then return the wp finger print info: [is_wp?,wp_ver]
-def wp_tracker_lookup(domain)
-  tracker=Wmap::WpTracker.new(:verbose=>false)
-	# first order
-	tracker.known_wp_sites.each do |key,val|
-		if key.include?(domain.strip.downcase) && val
-			ver=tracker.wp_ver(key)
-			tracker=nil
-			return [val,ver]
-		end
-	end
-  # second order
-  tracker.known_wp_sites.each do |key,val|
-    if key.include?(domain.strip.downcase) && key.include?("https") && val
-      tracker=nil
-      return [val,nil]
-    end
-  end
-  # third order
-  tracker.known_wp_sites.each do |key,val|
-    if key.include?(domain.strip.downcase)
-      tracker=nil
-      return [val,nil]
-    end
-  end
-  tracker=nil
-  return [nil,nil]
-end
-
-# perform the wpscan on a site
-def wpscan(domain)
-	url=site_tracker_lookup(domain)[0]
-	return nil if url.nil?
-	if url.include?("https")
-		command="wpscan --disable-tls-checks --ignore-main-redirect --url=" + url + " -o " + domain + ".wpscan"
-	else
-		command="wpscan --ignore-main-redirect --url=" + url + " -o " + domain + ".wpscan"
-	end
-	system(command)
-end
-
-puts Wmap.banner
-print_usage
-
-# open output file to write
-CSV.open("output.csv", "wb") do |csv|
-  cnt=1
-  # open RHPG input file to read
-  CSV.foreach(ARGV[0]) do |row|
-    puts "Processing row #{cnt}"
-    #puts row.inspect
-    my_row=Array.new
-    if cnt > 1
-      if is_domain?(row[0])
-=begin
-				if row[3] =~ /Keep/i && row[3] != /Redirect/i
-					unless File.exist?(row[0]+".wpscan")
-						wpscan(row[0])
-					end
-				end
-=end
-        my_row = row + site_tracker_lookup(row[0]) + wp_tracker_lookup(row[0])
-      else
-        my_row = row + [nil]*10
-      end
-    else
-      my_row = row + ["Website","Primary IP","Port","Hosting Status","Server","Response Code","MD5 Finger-print","Redirection","Timestamp", "WordPress", "WordPress Version"]
-    end
-    cnt+=1
-    csv << my_row
-  end
-  puts "All done. "
-end

data/bin/wmaps

@@ -1,23 +0,0 @@
-#!/usr/bin/env ruby
-# script to automate the new site discovery through by crawling all unique sites in the site store
-require "wmap"
-require "parallel"
-
-def wmap_worker(domain)
-   cmd = "wmap " + domain
-   puts "wmap discovery on domain: ", domain
-   system(cmd)
-end
-
-
-tracker=Wmap::DomainTracker.instance
-Parallel.map(tracker.known_internet_domains.keys, :in_processes => 10) { |target|
-  puts "Working on #{target} ..." if @verbose
-  wmap_worker(target)
-}
-=begin
-tracker.known_internet_domains.keys.map do |domain|
-  wmap_worker(domain)
-end
-=end
-tracker=nil

data/settings/tag_signatures

@@ -1,6 +0,0 @@
-# Adware signature file: signture string, description
-gtag.js, Google / DoubleClick Floodlight Tag
-analytics.js, Google Universal Analytics Tag
-ga.js, Google Analytics Tag
-utag.js, Tealium Tag
-all.js, Facebook Tag