wmap 2.5.1 → 2.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6f2042c146724dcfa9852bcb1920a2d9baded35fb8ba589d3af277277d678d36
-  data.tar.gz: d0ae5c5f90a4707eddbb91119b71ae2e9dc1c9ace6631e3f66e03d892ef3d8f1
+  metadata.gz: d6489f1302aecc4bc426c93c2e4f0a717e7f908186eee6ded623b3b0dbb10c0f
+  data.tar.gz: dc7875e566e929991dc9adce78db8dd5a47bc2643f745cea2c7b5a69f85ad8fb
 SHA512:
-  metadata.gz: 2a543f31d23e42604a0c86445eb3c2b469485527c0897a7495a8d74f593be19a1e915cf38e8ae03246012d5bb21734f8d2f93b0180863383a3a5260030fb1336
-  data.tar.gz: 830a9645c9633f0cd396cf9dfc40654a902ad3275ec37870f2fa1e06f64365bf9e0229f5acb0972af8569497c253e13d88b87c5d86710d324cb1510e3cd6679c
+  metadata.gz: d0520926107547b5870dd20a0fc9bbc4fdd8900708602e79db47edb37e1a9e0c82c4f98ea115c5a2d8c910c366d3ec1e91819f0cac5e83d52c3fac91f28a9cc9
+  data.tar.gz: 6358585417331756374da3f4a45a4a658cd4e59a5237bce9d2c8a0b8026683c678e49eac8f1b3e49a6f4359a2d9f8e1eb30d057424b801358f10a077f81fcd05

data/lib/wmap/url_crawler/adware_tag.rb

@@ -36,139 +36,129 @@ module Wmap
 
     # load the known tag signatures into an instance variable
   	def load_from_file (file, lc=true)
-  		begin
-        puts "Loading data file: #{file}"	if @verbose
-  			data_store=Hash.new
-  			f = File.open(file, 'r')
-  			f.each_line do |line|
-  				puts "Processing line: #{line}" if @verbose
-  				line=line.chomp.strip
-  				next if line.nil?
-  				next if line.empty?
-  				next if line =~ /^\s*#/
-  				line=line.downcase if lc==true
-  				entry=line.split(',')
-  				if data_store.key?(entry[0])
-  					next
-  				else
-  					data_store[entry[0]]=entry[1].strip
-  				end
+      puts "Loading data file: #{file}"	if @verbose
+			data_store=Hash.new
+			f = File.open(file, 'r')
+			f.each_line do |line|
+				puts "Processing line: #{line}" if @verbose
+				line=line.chomp.strip
+				next if line.nil?
+				next if line.empty?
+				next if line =~ /^\s*#/
+				line=line.downcase if lc==true
+				entry=line.split(',')
+				if data_store.key?(entry[0])
+					next
+				else
+					data_store[entry[0]]=entry[1].strip
+				end
 
-  			end
-  			f.close
-  			return data_store
-  		rescue => ee
-  			puts "Exception on method #{__method__}: #{ee}" if @verbose
-  			return nil
-  		end
+			end
+			f.close
+			return data_store
+		rescue => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+			return nil
   	end
 
     # load the known tag store cache into an instance variable
   	def load_tag_from_file (file, lc=true)
-  		begin
-        puts "Loading tag data file: #{file}"	if @verbose
-  			data_store=Hash.new
-  			f = File.open(file, 'r')
-  			f.each_line do |line|
-  				puts "Processing line: #{line}" if @verbose
-  				line=line.chomp.strip
-  				next if line.nil?
-  				next if line.empty?
-  				next if line =~ /^\s*#/
-  				line=line.downcase if lc==true
-  				entry=line.split(',')
-  				if data_store.key?(entry[0])
-  					next
-  				else
-  					data_store[entry[0]]=[entry[1].strip, entry[2].strip, entry[3]]
-  				end
-  			end
-  			f.close
-  			return data_store
-  		rescue => ee
-  			puts "Exception on method #{__method__}: #{ee}" if @verbose
-  			return nil
-  		end
+      puts "Loading tag data file: #{file}"	if @verbose
+			data_store=Hash.new
+			f = File.open(file, 'r')
+			f.each_line do |line|
+				puts "Processing line: #{line}" if @verbose
+				line=line.chomp.strip
+				next if line.nil?
+				next if line.empty?
+				next if line =~ /^\s*#/
+				line=line.downcase if lc==true
+				entry=line.split(',')
+				if data_store.key?(entry[0])
+					next
+				else
+					data_store[entry[0]]=[entry[1].strip, entry[2].strip, entry[3], entry[4]]
+				end
+			end
+			f.close
+			return data_store
+		rescue => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
+			return nil
   	end
 
     # Save the current tag store hash table into a file
   	def save_to_file!(file_tag=@tag_file, tags=@tag_store)
-      begin
-        puts "Saving the current wordpress site table from memory to file: #{file_tag} ..." if @verbose
-  			timestamp=Time.now
-  			f=File.open(file_tag, 'w')
-  			f.write "# Local tag file created by class #{self.class} method #{__method__} at: #{timestamp}\n"
-  			f.write "# Site, Landing URL, Detected Adware Tag, Tag Version, Tag Description\n"
-  			tags.each do |key, val|
-  				f.write "#{key}, #{val[0]}, #{val[1]}, #{val[2]}, #{val[3]}\n"
-  			end
-  			f.close
-  			puts "Tag store cache table is successfully saved: #{file_tag}"
-  		rescue => ee
-  			puts "Exception on method #{__method__}: #{ee}" if @verbose
-  		end
+      puts "Saving the current wordpress site table from memory to file: #{file_tag} ..." if @verbose
+			timestamp=Time.now
+			f=File.open(file_tag, 'w')
+			f.write "# Local tag file created by class #{self.class} method #{__method__} at: #{timestamp}\n"
+			f.write "# Site, Landing URL, Detected Adware Tag, Tag Version, Tag Description\n"
+			tags.each do |key, val|
+				f.write "#{key}, #{val[0]}, #{val[1]}, #{val[2]}, #{val[3]}\n"
+			end
+			f.close
+			puts "Tag store cache table is successfully saved: #{file_tag}"
+		rescue => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
   	end
   	alias_method :save!, :save_to_file!
 
     # add tag entries (from the sitetracker list)
   	def refresh (num=@max_parallel,use_cache=true)
-      #begin
-  		  puts "Add entries to the local cache table from site tracker: " if @verbose
-  			results=Hash.new
-  			tags=Wmap::SiteTracker.instance.known_sites.keys
-  			if tags.size > 0
-  				Parallel.map(tags, :in_processes => num) { |target|
-  					check_adware(target,use_cache)
-  				}.each do |process|
-  					if !process
-  						next
-  					else
-  						results.merge!(process)
-  					end
-  				end
-  				@tag_store.merge!(results)
-  				puts "Done loading entries."
-          tags=nil
-  				return results
-  			else
-  				puts "Error: no entry is loaded. Please check your list and try again."
-  			end
+		  puts "Add entries to the local cache table from site tracker: " if @verbose
+			results=Hash.new
+			tags=Wmap::SiteTracker.instance.known_sites.keys
+			if tags.size > 0
+				Parallel.map(tags, :in_processes => num) { |target|
+					check_adware(target,use_cache)
+				}.each do |process|
+					if !process
+						next
+					else
+						results.merge!(process)
+					end
+				end
+				@tag_store.merge!(results)
+				puts "Done loading entries."
         tags=nil
-  			return results
-  		#rescue => ee
-  		#	puts "Exception on method #{__method__}: #{ee}" if @verbose
-  		#end
+				return results
+			else
+				puts "Error: no entry is loaded. Please check your list and try again."
+			end
+      tags=nil
+			return results
+		rescue => ee
+			puts "Exception on method #{__method__}: #{ee}" if @verbose
   	end
 
     # Give a  site, locate the landing page, then sift out the adware tag if found
   	def check_adware(site,use_cache=true)
-      #begin
-  		  puts "Check the site for known Adware tags: #{site}" if @verbose
-        record = Hash.new
-  			if use_cache && @tag_store.key?(site)
-				  puts "Site entry already exist. Skipping: #{site}" if @verbose
-  			else
-          url = fast_landing(site)
-          tags = find_tags(url)
-          return record if tags.size==0
-          tag_vers=tags.map do |tag|
-            get_ver(url,tag)
-          end
-          tag_descs=tags.map do |tag|
-            Base64.urlsafe_encode64(get_desc(url,tag))
-          end
-  				if tags
-            record[site]=[url, tags.join("|"), tag_vers.join("|"), tag_descs.join("|")]
-            @tag_store.merge!(record)
-            puts "Tag entry loaded: #{record}" if @verbose
-          else
-            puts "No tag found. Skip site #{site}" if @verbose
-          end
-  			end
-        return record
-      #rescue => ee
-  		#	puts "Exception on method #{__method__}: #{ee}: #{site}" if @verbose
-  		#end
+		  puts "Check the site for known Adware tags: #{site}" if @verbose
+      record = Hash.new
+			if use_cache && @tag_store.key?(site)
+			  puts "Site entry already exist. Skipping: #{site}" if @verbose
+			else
+        url = fast_landing(site)
+        tags = find_tags(url)
+        return record if tags.size==0
+        tag_vers=tags.map do |tag|
+          get_ver(url,tag)
+        end
+        tag_descs=tags.map do |tag|
+          Base64.urlsafe_encode64(get_desc(url,tag))
+        end
+				if tags
+          record[site]=[url, tags.join("|"), tag_vers.join("|"), tag_descs.join("|")]
+          @tag_store.merge!(record)
+          puts "Tag entry loaded: #{record}" if @verbose
+        else
+          puts "No tag found. Skip site #{site}" if @verbose
+        end
+			end
+      return record
+    rescue => ee
+			puts "Exception on method #{__method__}: #{ee}: #{site}" if @verbose
   	end
 
     # Given a site, determine the landing url
@@ -191,26 +181,26 @@ module Wmap
       end
       puts "Landing url found: #{url}" if @verbose
       return url
+    rescue => ee
+      puts "Exception on method #{__method__}: #{ee}" if @verbose
     end
 
     # Search the page for known tag signatures. If found return them in an array
   	def find_tags(url)
-  		begin
-  			puts "Search and return tags within the url payload: #{url}" if @verbose
-  			tag_list = []
-        doc = Nokogiri::HTML(open(url))
-        doc.text.each_line do |line|
-          my_line = line.downcase
-          @tag_signatures.keys.map do |tag|
-            tag_list.push(tag) if my_line.include?(tag)
-          end
+			puts "Search and return tags within the url payload: #{url}" if @verbose
+			tag_list = []
+      doc = Nokogiri::HTML(open(url))
+      doc.text.each_line do |line|
+        my_line = line.downcase
+        @tag_signatures.keys.map do |tag|
+          tag_list.push(tag) if my_line.include?(tag)
         end
-        doc = nil
-        return tag_list
-      rescue => ee
-        puts "Exception on method #{__method__}: #{ee}" if @verbose
-        return []
-  		end
+      end
+      doc = nil
+      return tag_list
+    rescue => ee
+      puts "Exception on method #{__method__}: #{ee}" if @verbose
+      return []
     end
 
     # Search the url payload for known tag version identifier. If found return a string, else empty string.
@@ -256,6 +246,9 @@ module Wmap
       end
       doc = nil
       return tag_ver
+    rescue => ee
+      puts "Exception on method #{__method__}: #{ee}: #{url} : #{tag}" if @verbose
+      return tag_ver
     end
 
     # Search the url payload for known tag. If found return the base64 encode whole script snippet.
@@ -272,6 +265,9 @@ module Wmap
       end
       doc = nil
       return tag_desc
+    rescue => ee
+      puts "Exception on method #{__method__}: #{ee}: #{url}: #{tag}" if @verbose
+      return tag_desc
     end
 
 

data/version.txt

@@ -3,8 +3,8 @@
 ###############################################################################
 package = wmap
 # wmap version 2.0 == web_discovery version 1.5.3
-version = 2.5.1
-date = 2019-03-17
+version = 2.5.2
+date = 2019-03-18
 
 author = Sam (Yang) Li
 email = yang.li@owasp.org

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wmap
 version: !ruby/object:Gem::Version
-  version: 2.5.1
+  version: 2.5.2
 platform: ruby
 authors:
 - Sam (Yang) Li
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-17 00:00:00.000000000 Z
+date: 2019-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dnsruby