wmap 2.4.8 → 2.4.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '039be636ec3179df0106265ea90c9ac86a27f34c1291564a266f97d03105c86c'
-  data.tar.gz: 73453c025547726381e0effd66bae71977b1a242952a6b83e6e8ba7c5d81378f
+  metadata.gz: 7df3c5698974ccb73832bafefc264aebf82addba523ce2b5a2804684977bc558
+  data.tar.gz: 16546e8b0daf8b16dd0bb4a89815569a3dbf058bd9ec6cddc3341a1c33640e91
 SHA512:
-  metadata.gz: bdfd196feab5fcd300a8b7856ca1731f90e799cb13ae1545a81aa5ababc859131fdcd9f6f522b6945de200e312e661d89e1cd824e0d0f4587a2d8c7c1e49fe9a
-  data.tar.gz: 27a27879fed90a64f9ff71407388b6e8cbc1ad6e6c8d67964cdfed8a03e729f2a725ed69b738c2c6636912256f3e9fb3a32fd74e923d193b7d5ea5ad290bafa4
+  metadata.gz: f089db7404c7bb058aedf90ae195cf37768482ba1d08f099a1b3cc85c7e0d0510552adce9546c597053aa6440f378fecc561e034a352914780d04408be5eb04b
+  data.tar.gz: a7a74b5527d09296de0c33864dc680a10b8454d7b7545b3f1f803b3245c321cc708e384197cc469dabf5d4007fea3cb0bdfd11274ab13188853b66f053817a97

data/bin/googleBot

@@ -14,7 +14,7 @@ Wmap.wlog("Execute the command: googleBot","googleBot",Log_dir+"wmap.log")
 abort "Incorrect program argument - no argument needed! Proper Usage: googleBot " unless ARGV.length==0
 
 puts "Start the Google search scraper. This may take a while, please be patient ..."
-dis=Wmap::GoogleSearchScraper.new
+dis=Wmap::GoogleSearchScraper.new(:verbose=>false)
 dis.workers
 sites=dis.discovered_sites_from_scraper.keys
 puts "Update Google search results into the data repository:"

data/lib/wmap/host_tracker.rb

@@ -116,7 +116,7 @@ class Wmap::HostTracker
 					puts "Domain root: #{root}" if @verbose
 					domain_tracker=Wmap::DomainTracker.instance
 					domain_tracker.data_dir=@data_dir
-					if domain_tracker.instance.domain_known?(root)
+					if domain_tracker.domain_known?(root)
 						domain_tracker=nil
 						record[host]=ip
 						record[ip]=host

data/lib/wmap/site_tracker.rb

@@ -103,11 +103,14 @@ class Wmap::SiteTracker
 
 	# Setter to add site entry to the cache one at a time
 	def add(site)
-		puts "Add entry to the site store: #{site}"
-		begin
+		#begin
+			puts "Add entry to the site store: #{site}"
 			# Preliminary sanity check
 			site=site.strip.downcase unless site.nil?
-			raise "Site is already exist. Skip #{site}" if site_known?(site)
+			if site_known?(site)
+				puts  "Site is already exist. Skip #{site}"
+				return nil
+			end
 			site=normalize_url(site) if is_url?(site)
 			site=url_2_site(site) if is_url?(site)
 			puts "Site in standard format: #{site}" if @verbose
@@ -233,11 +236,11 @@ class Wmap::SiteTracker
 				my_tracker=nil
 				return nil
 			end
-		rescue => ee
-			puts "Exception on method #{__method__}: #{ee}"
-			deact=nil
-			return nil
-		end
+		#rescue => ee
+		#	puts "Exception on method #{__method__}: #{ee}"
+		#	deact=nil
+		#	return nil
+		#end
 	end
 
 	# Setter to add site entry to the cache table in batch (from a file)

data/logs/wmap.log

@@ -1544,3 +1544,9 @@
 2019-02-17 22:09:16 -0500: wmap: Execute the command: wmap penguinrandomhouse.com
 2019-02-17 22:11:57 -0500: wmap: Execute the command: wmap /tmp/test
 2019-02-17 22:50:46 -0500: wmap: Execute the command: wmap /tmp/test1
+2019-02-20 20:41:31 -0500: googleBot: Execute the command: googleBot
+2019-02-20 20:56:04 -0500: googleBot: Execute the command: googleBot
+2019-02-20 21:05:21 -0500: googleBot: Execute the command: googleBot
+2019-02-20 21:10:59 -0500: googleBot: Execute the command: googleBot
+2019-02-20 21:17:21 -0500: googleBot: Execute the command: googleBot
+2019-02-20 21:26:13 -0500: googleBot: Execute the command: googleBot

data/settings/google_keywords.txt

@@ -2,8 +2,19 @@
 # Used by the Wmap::GoogleSearchScraper class
 
 # Official whole name
-Google Inc.
+Penguin Random House
 
 # 2014 purchase
-DeepMind Technologies
+Random House
 
+# Parent company
+Bertelsmann
+
+# Division
+Crown Books
+
+# Division
+Dorling Kindersley
+
+# Division
+Penguin Random House Canada

data/version.txt

@@ -3,8 +3,8 @@
 ###############################################################################
 package = wmap
 # wmap version 2.0 == web_discovery version 1.5.3
-version = 2.4.8
-date = 2019-02-18
+version = 2.4.9
+date = 2019-02-20
 
 author = Sam (Yang) Li
 email = yang.li@owasp.org

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wmap
 version: !ruby/object:Gem::Version
-  version: 2.4.8
+  version: 2.4.9
 platform: ruby
 authors:
 - Sam (Yang) Li
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-18 00:00:00.000000000 Z
+date: 2019-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dnsruby