wmap 2.5.2 → 2.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d6489f1302aecc4bc426c93c2e4f0a717e7f908186eee6ded623b3b0dbb10c0f
-  data.tar.gz: dc7875e566e929991dc9adce78db8dd5a47bc2643f745cea2c7b5a69f85ad8fb
+  metadata.gz: ae590c812b4919eb1b66841f6eb873de93727cbf8847fe6392dc19c94e4ef158
+  data.tar.gz: 7530a5c491a7d87289e41c4ef4b0df6f201495c107270892e5e6d22f9584da9b
 SHA512:
-  metadata.gz: d0520926107547b5870dd20a0fc9bbc4fdd8900708602e79db47edb37e1a9e0c82c4f98ea115c5a2d8c910c366d3ec1e91819f0cac5e83d52c3fac91f28a9cc9
-  data.tar.gz: 6358585417331756374da3f4a45a4a658cd4e59a5237bce9d2c8a0b8026683c678e49eac8f1b3e49a6f4359a2d9f8e1eb30d057424b801358f10a077f81fcd05
+  metadata.gz: fa9211e5e14e74a24266677b8993071f9be1e75b9387d35a36233192c885521abbb8ecd31b0b0dc2d65db4de50cb0513e0a4cd214bed18672a1cf3b1b9cd52f6
+  data.tar.gz: 70d6dc893be0273b16b4f8dd78d9fe4f3ec7bf5310ea0c8b299100535300053a19f8d936ac6422cffd5251d0dc2fae991eeaa41ac31f0a639d52994bf9a5d645

data/bin/RHPG

@@ -11,7 +11,7 @@ def print_usage
 end
 
 # Lookup the site store for a domain; then return the fingger print info of the site
-def site_lookup(domain)
+def site_tracker_lookup(domain)
   tracker=Wmap::SiteTracker.instance
 	tracker.verbose=false
   #first order search
@@ -33,7 +33,7 @@ def site_lookup(domain)
 end
 
 # look up the wp site data store for a domain; then return the wp finger print info: [is_wp?,wp_ver]
-def wp_site_lookup(domain)
+def wp_tracker_lookup(domain)
   tracker=Wmap::WpTracker.new(:verbose=>false)
 	# first order
 	tracker.known_wp_sites.each do |key,val|
@@ -63,7 +63,7 @@ end
 
 # perform the wpscan on a site
 def wpscan(domain)
-	url=site_lookup(domain)[0]
+	url=site_tracker_lookup(domain)[0]
 	return nil if url.nil?
 	if url.include?("https")
 		command="wpscan --disable-tls-checks --ignore-main-redirect --url=" + url + " -o " + domain + ".wpscan"
@@ -93,7 +93,7 @@ CSV.open("output.csv", "wb") do |csv|
 					end
 				end
 =end
-        my_row = row + site_lookup(row[0]) + wp_site_lookup(row[0])
+        my_row = row + site_tracker_lookup(row[0]) + wp_tracker_lookup(row[0])
       else
         my_row = row + [nil]*10
       end

data/bin/trusts

@@ -11,7 +11,7 @@ print_usage
 Log_dir=File.dirname(__FILE__)+'/../logs/'
 Wmap.wlog("Execute the command: trust #{ARGV[0]}","trust",Log_dir+"wmap.log")
 
-dt=Wmap::DomainTracker.instance.new
+dt=Wmap::DomainTracker.instance
 ct=Wmap::CidrTracker.new(:verbose=>true)
 abort "Incorrect program argument! Proper usage: trust [domain | netblock]" unless ARGV.length==1 && (File.exist?(ARGV[0]))
 

data/dicts/ccsld.txt

@@ -3,7 +3,7 @@
 #    by Yang Li, Last modification date: 05/12/2013
 
 Afghanistan
-Domain registrations at the second-level are unrestricted, but more expensive. Third-level names have restrictions based on which second-level name they are beneath 
+Domain registrations at the second-level are unrestricted, but more expensive. Third-level names have restrictions based on which second-level name they are beneath
 
     .af - Registrations are taken directly at the second level or at the third level beneath the following domain extensions.
     .com.af - Commercial entities (Trade license / Business authorization or approval from the Ministry of Commerce.)
@@ -13,12 +13,12 @@ Domain registrations at the second-level are unrestricted, but more expensive. T
     .org.af - Non-commercial entities
 
 Aland Islands
-The country domain extension for the �land Islands is .ax. Registrants must be connected with the �land Islands.
+The country domain extension for the �land Islands is .ax. Registrants must be connected with the �land Islands.
 
     .ax - Registrations are taken directly at the second-level.
 
 
-Note: Previously, most �land Islands websites were registered at the third level under the aland.fi subdomain (�land Islands are part of Finland).
+Note: Previously, most �land Islands websites were registered at the third level under the aland.fi subdomain (�land Islands are part of Finland).
 Albania
 These are the country extensions for Albania. Registrants must be an organization with legal representative in Albania. There is a limit of one domain per organization, and it must be under an appropriate subdomain. Registrations are made at the third-level, beneath one of the following subdomains.
 
@@ -48,7 +48,7 @@ The country domain extension for American Samoa is .as. There is no restriction
     .as - Registrations are taken directly at second level.
 
 Andorra
-Registrations must be pre-approved by Office for State Emblems and Signs, are generally limited to residents of Andorra or Andorran trademark holders with an agent in Andorra, and the number of registrations per person is limited. 
+Registrations must be pre-approved by Office for State Emblems and Signs, are generally limited to residents of Andorra or Andorran trademark holders with an agent in Andorra, and the number of registrations per person is limited.
 
     .ad - Country code top-level domain (ccTLD) for Andorra. Names can be registered directly at the second level (i.e. example.ad).
     .nom.ad - Country domain extension for personal websites.
@@ -75,7 +75,7 @@ These are the country domain extensions for Anguilla. There are no restrictions
 Antarctica
 The country domain for Antarctica is .aq. This top-level domain is reserved for organizations that conduct work in Antarctica or promote the Antarctic and Southern Ocean regions. Must have physical presence in Antarctica or be a governmental organization signatory to the Antarctic Treaty.
 
-    .aq - Registrations are taken at the second level. 
+    .aq - Registrations are taken at the second level.
 
 Antigua and Barbuda
 These are the country domain extensions for Antigua and Barbuda. There are no restrictions on registering domain names with the .ag extension.
@@ -105,7 +105,7 @@ Although domain names must be registered at the third level, some public and gov
 Armenia
 The top-level country code for Armenia is .am. Some generic names are reserved. Except for reserved names like .com.am, .net.am, .org.am and others, any person in the world can register a .am domain for a fee. Registrations are subject to review.
 
-    .am  - Registrations are made directly at the second-level. 
+    .am  - Registrations are made directly at the second-level.
 
 Aruba
 The country domain extension for Aruba is .aw. Registrations are directly at second level, or beneath the .com.aw subdomain.
@@ -290,7 +290,7 @@ The top-level domain for Bhutan is .bt. Registrants must have a local presence i
 Bolivia
 Here are the country domain extensions for Bolivia. There are no restrictions on who can register .bo domains.
 
-    .bo - Can be registered directly at the second-level, but this is more expensive. 
+    .bo - Can be registered directly at the second-level, but this is more expensive.
     .com.bo - Commercial entities
     .net.bo - Network service providers
     .org.bo - Organizations
@@ -308,11 +308,11 @@ Here are the domain extensions for Bosnia and Herzegovina. To register a domain
     .org.ba - Non-profit organizations
     .net.ba - Telecommunications providers
     .edu.ba - Education
-    .gov.ba - Any of the governments (local, regional or national) 
+    .gov.ba - Any of the governments (local, regional or national)
     .mil.ba - Military
     .unsa.ba - University of Sarajevo
     .untz.ba - University of Tuzla
-    .unmo.ba - University of Mostar "D�emal Bijedic"
+    .unmo.ba - University of Mostar "D�emal Bijedic"
     .unbi.ba - University of Bihac
     .unze.ba - University of Zenica
     .co.ba - Maintained by a company called Sayber, located in Sarajevo.
@@ -438,7 +438,7 @@ Here are the domain extensions for Burma. Note that, although there are some fun
 Burundi
 Here is the top-level domain for Burundi. Anyone in the world can register a .bi domain name, however, use for misleading purposes is discouraged.
 
-    .bi  - Registrations are taken directly at second-level or at third-level beneath one of several second-level domains. 
+    .bi  - Registrations are taken directly at second-level or at third-level beneath one of several second-level domains.
 
 Cambodia
 Here are the country code extensions for Cambodia. Note that offensive or inappropriate names may be refused registration. Registrations are at third level beneath the following second-level labels.
@@ -460,7 +460,7 @@ Here are the country domain extensions for Cameroon. Local presence is required
 Canada
 Here are the country domain extensions for Canada. To register a .ca domain name, you must meet the Canadian Presence Requirements as specified by the registry.
 
-    .ca - registrants can either register their domain names directly at the second-level, or at the third-level, under one of the (geographic) second-level domains below. 
+    .ca - registrants can either register their domain names directly at the second-level, or at the third-level, under one of the (geographic) second-level domains below.
     .ab.ca - Alberta
     .bc.ca - British Columbia
     .mb.ca - Manitoba
@@ -481,7 +481,7 @@ Examples of entities that meet the Canadian Presence Requirements include:
     a Canadian citizen of the age of majority,
     a permanent resident of Canada,
     a legally recognized Canadian organization,
-    an Inuit, First Nation, M�tis or other people indigenous to Canada,
+    an Inuit, First Nation, M�tis or other people indigenous to Canada,
     an Indian Band as defined in the Indian Act of Canada,
     a foreign resident of Canada that holds a registered Canadian trademark,
     An executor, administrator or other legal representative of a person or organization that meets the requirements,
@@ -508,15 +508,15 @@ Here's the top-level domain for the Central African Republic. It is intended for
 
     .cf - Domain registrations are done directly at the second-level
 
-Chad (French: Tchad) 
+Chad (French: Tchad)
 Here is the top-level domain for Chad. There are no restrictions on registering domain names with this extension.
 
     .td - Registrations are made directly at the second-level. Note that registrations were not accepted by the registry at the time of writing.
 
 Chile
-Here are the domain extensions for Chile. Anyone can register a .cl domain name, however a foreign registration must have a domestic representative who has a RUT (RUT, which stands for Rol �nico Tributario, is the Chilean national identification number).
+Here are the domain extensions for Chile. Anyone can register a .cl domain name, however a foreign registration must have a domestic representative who has a RUT (RUT, which stands for Rol �nico Tributario, is the Chilean national identification number).
 
-    .cl - Registrations are made directly at the second level (except for government sites, as indicated below). 
+    .cl - Registrations are made directly at the second level (except for government sites, as indicated below).
     .gov.cl - For government websites.
     .gob.cl - For government websites.
 
@@ -617,7 +617,7 @@ Here's the top-level domain for the Democratic Republic of the Congo. There is n
 Congo, Republic of the
 Here's the top-level domain for the Republic of Congo. Registration is unrestricted - anyone can register a domain name under this extension. Republic of the Congo citizens get one free domain name registration.
 
-    .cg - Registration is done directly at the second-level. 
+    .cg - Registration is done directly at the second-level.
 
 Cook Islands
 Here are the domain extensions for the Cook Islands.
@@ -647,8 +647,8 @@ Here are the domain name extensions for Costa Rica. There are varying restrictio
     .or.cr - Non-profit organizations
     .sa.cr - Health related institutions
 
-C�te d'Ivoire
-Here's the top-level domain for C�te d'Ivoire. Registrants name must match official name of company, organization, or trademark.
+C�te d'Ivoire
+Here's the top-level domain for C�te d'Ivoire. Registrants name must match official name of company, organization, or trademark.
 
     .ci - Registrations are made directly at the second level, or at third level beneath some second-level domain.
 
@@ -658,9 +658,9 @@ Here's the country code top-level domain for Croatia. Registration is limited to
     .hr - Domain registrations are taken directly at the second level, or under various subdomains such as .com.hr and .iz.hr.
 
 Cuba
-Here's the top-level domain for Cuba. Registrations are limited to people and companies located in Cuba. Also, third-level registrations may have other limitations based on which second-level domain they are within. 
+Here's the top-level domain for Cuba. Registrations are limited to people and companies located in Cuba. Also, third-level registrations may have other limitations based on which second-level domain they are within.
 
-    .cu - May register at second-level or at the third-level, beneath generic second-level domains. 
+    .cu - May register at second-level or at the third-level, beneath generic second-level domains.
 
 Cyprus, Republic of
 Limited to Cyprus residents and registered companies/organizations. There are further restrictions for different subdomains
@@ -682,17 +682,17 @@ Limited to Cyprus residents and registered companies/organizations. There are fu
 Czech Republic
 Here is the domain extension for the Czech Republic. Businesses must provide tax registration number when registering a domain name with this extension.
 
-    .cz - Domain names can be registered directly at second level. 
+    .cz - Domain names can be registered directly at second level.
 
 Denmark
 Here's the top-level domain for Denmark. There are no restrictions on who can register domain names with this extension.
 
-    .dk - Domain names can be registered directly at second level. 
+    .dk - Domain names can be registered directly at second level.
 
 Djibouti
 Here's the top-level domain for Djbouti. There are no restrictions on who can register domain names with this extension, although some names are reserved or subject to famous trademark owners having first registration rights.
 
-    .dj - Domain names can be registered directly at second level. 
+    .dj - Domain names can be registered directly at second level.
 
 Dominica
 Here are the domain extensions for Dominica. There are no restrictions on who can register domain names with this extension. Third-level names in .com.dm, .net.dm, and .org.dm are included automatically upon registration of all .dm domain names.
@@ -865,7 +865,7 @@ Here are the country domain extensions for French Polynesia. Local presence is r
     .pf - Registrations are accepted directly at the second-level or beneath the following subdomain.
     .com.pf
 
-French Southern and Antarctic Lands (France) 
+French Southern and Antarctic Lands (France)
 Here is the top-level domain for French Southern and Antarctic Lands. A local administrative contact with proof of address in French Southern Territory is required.
 
     .tf - Registrations are made directly at the second-level. Note that an unofficial service makes free, redirected domains available at third level.
@@ -888,7 +888,7 @@ Here is the country code top-level domain for Georgia. Domain name registrations
 Germany
 Here's the top-level domain for Germany. Registrants must have administrative contact resident in Germany.
 
-    .de - domain names can be registered directly at second level. 
+    .de - domain names can be registered directly at second level.
 
 Ghana
 Below are the domain extensions for Ghana. Companies wishing to use this extension must be registered in Ghana, and the domain administrator must be located there. Second-level registrations are no longer accepted. Registrations must now be at third level beneath second-level labels..
@@ -925,7 +925,7 @@ Here is the country code top-level domain for Grenada. There are no restrictions
     .gd - Domain names are registered directly at the second level.
 
 Guadeloupe
-Below are the domain extensions for Guadeloupe. Domain names with this extension is available worldwide but more expensive for non-Guadeloupe residents. 
+Below are the domain extensions for Guadeloupe. Domain names with this extension is available worldwide but more expensive for non-Guadeloupe residents.
 
     .gp - Registrations are directly at second level, or at third level beneath the following second-level categories.
     .com.gp
@@ -1096,7 +1096,7 @@ Below are the country domain extensions for Iran. These domains are generally re
     .sch.ir - Schools, primary and secondary education
 
 Iraq
-Here are the domain extensions for Iraq. No online registrations are known to be taking place at the time of writing. 
+Here are the domain extensions for Iraq. No online registrations are known to be taking place at the time of writing.
 
     .iq - General
     .gov.iq - Governmental entities
@@ -1138,7 +1138,7 @@ Below are the country domain extensions for Isle of Man. Restrictions apply to s
 Italy
 Here's the country code top-level domain for Italy.  Registrants must be a resident of an EU country to register a domain name with this extension. Registration requires sending a paper or fax letter which must be completed in Italian.
 
-    .it - Registrations are at second level or third level under various second-level domains. Most second-level domains represent a province, region, or municipality. There is also .gov.it for government organizations.  
+    .it - Registrations are at second level or third level under various second-level domains. Most second-level domains represent a province, region, or municipality. There is also .gov.it for government organizations.
 
 Jamaica
 Below are the country domain extensions for Jamaica. There are no restrictions on who can register domain names with this extension, however, it is more expensive for non-residents. It is also cheaper for government departments and non-profit organizations. And it is free for schools and educational institutions.
@@ -1238,7 +1238,7 @@ Here are the domain extensions for South Korea. Registrants must have a local pr
     .re.kr - Research (Organizations or individuals)
     .pe.kr - Personal (Individuals)
     .go.kr - Government
-    .mil.kr - Military    
+    .mil.kr - Military
     .ac.kr - Colleges or universities
     .hs.kr - High schools
     .ms.kr - Middle schools
@@ -1248,7 +1248,7 @@ Here are the domain extensions for South Korea. Registrants must have a local pr
 
 Organizations or individuals that have connection with relevant areas:
 
-    .seoul.kr - Seoul    
+    .seoul.kr - Seoul
     .busan.kr - Busan
     .daegu.kr - Daegu
     .incheon.kr - Incheon
@@ -1829,10 +1829,10 @@ Below are the country domain extensions for Poland. There are no restrictions fo
     .warszawa.pl - Warsaw
     .wroc.pl - Wroclaw
     .wroclaw.pl - Wroclaw
-    .krakow.pl - Krak�w
+    .krakow.pl - Krak�w
     .katowice.pl - Katowice
     .poznan.pl - Poznan
-    .lodz.pl - L�dz
+    .lodz.pl - L�dz
     .gda.pl - Gdansk
     .gdansk.pl - Gdansk
     .slupsk.pl - Slupsk
@@ -1842,8 +1842,8 @@ Below are the country domain extensions for Poland. There are no restrictions fo
     .bialystok.pl - Bialystok
     .olsztyn.pl - Olsztyn
     .torun.pl - Torun
-    .gorzow.pl - Gorz�w Wielkopolski
-    .zgora.pl - Zielona G�ra
+    .gorzow.pl - Gorz�w Wielkopolski
+    .zgora.pl - Zielona G�ra
 
 Portugal
 Below are the domain extensions for Portugal. There are various restrictions for each subdomain, however the .com.pt extension is unrestricted.
@@ -1873,7 +1873,7 @@ Here are the country domain extensions for Puerto Rico. Some subdomains have res
     .org.pr - For organizations (but not restricted to)
     .pro.pr - For professionals
     .est.pr - For university students
-    .prof.pr - For university professors 
+    .prof.pr - For university professors
     .ac.pr - For academics
 
 Qatar
@@ -1907,8 +1907,8 @@ Below are the country domain extensions for Romania. There are no restrictions f
     .tm.ro
     .www.ro
 
-R�union
-Here are the country domain extensions for R�union. Registrants are required to have local presence.
+R�union
+Here are the country domain extensions for R�union. Registrants are required to have local presence.
 
     .re - Domain names are registered directly at the second-level and at the third-level under the following subdomains.
     .asso.re - Associations
@@ -2002,19 +2002,19 @@ Here is the top-level domain for San Marino. The domain name must be the same as
 
     .sm - Domain name registrations are made directly at the second-level.
 
-S�o Tom� and Pr�ncipe
-Here is the top-level domain for S�o Tom� and Pr�ncipe. There are no restrictions on registering domain names with this extension.
+S�o Tom� and Pr�ncipe
+Here is the top-level domain for S�o Tom� and Pr�ncipe. There are no restrictions on registering domain names with this extension.
 
     .st - Registrations are made directly at the second-level, although the following second-level domains have been reserved for future use..
 
 Reserved Second-Level Domains
 The following second-level domains have been reserved by the registry for future use.
 
-    .gov.st - Government of S�o Tom� and Pr�ncipe
-    .saotome.st - Island of S�o Tom�
+    .gov.st - Government of S�o Tom� and Pr�ncipe
+    .saotome.st - Island of S�o Tom�
     .principe.st - Island of Principe
-    .consulado.st - S�o Tom� and Pr�ncipe consulates
-    .embaixada.st - S�o Tom� and Pr�ncipe embassies
+    .consulado.st - S�o Tom� and Pr�ncipe consulates
+    .embaixada.st - S�o Tom� and Pr�ncipe embassies
     .org.st
     .edu.st
     .net.st
@@ -2220,29 +2220,29 @@ Below are the domain extensions for Sweden. Some third-level domains are restric
     .se - Registrations can be made directly at the second-level or under the following subdomains.
     .a.se - Stockholm City
     .b.se - Stockholm County
-    .ac.se - V�sterbotten County
+    .ac.se - V�sterbotten County
     .bd.se - Norrbotten County
     .c.se - Uppsala County
-    .d.se - S�dermanland County
-    .e.se - �sterg�tland County
-    .f.se - J�nk�ping County
+    .d.se - S�dermanland County
+    .e.se - �sterg�tland County
+    .f.se - J�nk�ping County
     .g.se - Kronoberg County
     .h.se - Kalmar County
     .i.se - Gotland County
     .k.se - Blekinge County
     .l.se - Kristianstad County
-    .m.se - Sk�ne County
+    .m.se - Sk�ne County
     .n.se - Halland County
-    .o.se - V�stra G�taland County
-    .p.se - �lvsborg County
+    .o.se - V�stra G�taland County
+    .p.se - �lvsborg County
     .r.se - Skaraborg County
-    .s.se - V�rmland County
-    .t.se - �rebro County
-    .u.se - V�stmanland County
+    .s.se - V�rmland County
+    .t.se - �rebro County
+    .u.se - V�stmanland County
     .w.se - Dalarna County
-    .x.se - G�vleborg County
-    .y.se - V�sternorrland County
-    .z.se - J�mtland County
+    .x.se - G�vleborg County
+    .y.se - V�sternorrland County
+    .z.se - J�mtland County
     .org.se - Non-profit Organizations
     .pp.se - Private Persons
     .tm.se - Trade Marks
@@ -2270,7 +2270,7 @@ Below are the domain extensions for Syria. There are no restrictions on register
 Taiwan (Republic of China)
 Below are the country domain extensions for Taiwan. Requirements vary depending on which second-level name registration is within. Non-residents are allowed to register domain names under several subdomains.
 
-    .tw - Unrestricted. Registrations are made directly at second level and beneath the following subdomains. 
+    .tw - Unrestricted. Registrations are made directly at second level and beneath the following subdomains.
     .edu.tw - For educational and academic institutions
     .gov.tw - For Republic of China government agencies
     .mil.tw - For the Military of the Republic of China
@@ -2423,7 +2423,7 @@ Here is the top-level domain for Turks and Caicos Islands. There are no restrict
 Tuvalu
 Here is the country code top-level domain for the island nation of Tuvalu. There are no restrictions on the second-level domain names, however, some second-level names are reserved (i.e. .com.tv, .net.tv, .org.tv, .gov.tv). The .tv domain has been a popular choice for websites for television shows.
 
-    .tv - Registrations are directly at the second-level. 
+    .tv - Registrations are directly at the second-level.
 
 Uganda
 Below are the domain extensions for Uganda. There are no restrictions on registering domain names with this extension.
@@ -2450,7 +2450,7 @@ Here are the domain extensions for Ukraine. To register a domain name, the regis
 United Arab Emirates
 These are the country domain extensions for the United Arab Emirates. Second-level registrations are generally unrestricted. Third-level registrations require local presence and documentation (business registration), and if applicable, certificate from relevant ministry (for org, mil, gov, sch and ac).
 
-    .ae - Companies, Organizations or Individuals. Domain names can be registered directly at the second-level or at the third-level, under one of the following extensions. 
+    .ae - Companies, Organizations or Individuals. Domain names can be registered directly at the second-level or at the third-level, under one of the following extensions.
     .co.ae - Companies -- was deprecated by UAEnic in 2003 and reintroduced by .aeDA (domain registrar for the .ae ccTLD) in 2008
     .net.ae - Network Providers
     .gov.ae - Government and Ministries
@@ -2477,6 +2477,9 @@ Below are the country domain extensions for the United Kingdom (UK). There are n
     .plc.uk - Public limited companies
     .police.uk - Police
     .sch.uk - Local education authorities, schools, primary and secondary education, community education
+    .gb.net - UK specific
+    .uk.com - UK specific
+    .uk.net - UK specific 
 
 United States of America
 Below are the country domain extensions for the United States. Registrants of .us domains must be United States citizens, residents, or organizations, or a foreign entity with a presence in the United States. Originally registrations were within third or fourth level hierarchy, but currently direct second-level registrations are allowed. Also, the kids.us extension allows third-level registrations for child-friendly sites.
@@ -2630,7 +2633,7 @@ Here are the domain extensions for Yemen. Registrants must have a local contact
     .net.ye - Network Providers
     .org.ye - Non-commercial Organizations
     .plc.ye
-    .gov.ye - Government and Governmental System 
+    .gov.ye - Government and Governmental System
 
 Zambia
 Here are the domain extensions for Zambia. Local presence is required to register domain names with this extension.

data/lib/wmap/url_checker.rb

@@ -32,282 +32,266 @@ class Wmap::UrlChecker
 
 	# Main worker method to perform various checks on the URL / site
 	def url_worker (url)
-		begin
-			puts "Checking out an unknown URL: #{url}" if @verbose
-			url=url.strip.downcase
-			raise "Invalid URL format: #{url}" unless is_url?(url)
-			timestamp=Time.now
-			host=url_2_host(url)
-			ip=host_2_ip(host)
-			port=url_2_port(url)
-			code=10000
-			if @url_code.key?(url)
-				code=@url_code[url]
-			else
-				code=response_code(url)
-			end
-			if code>=300 && code < 400
-				loc=landing_location(4,url)
-			else
-				loc=nil
-			end
-			if @url_finger_print.key?(url)
-				fp=@url_finger_print[url]
-			else
-				fp=response_body_md5(url)
-			end
-			if @url_server.key?(url)
-				server=@url_server[url]
-			else
-				server=get_server_header(url)
-			end
-			# save the data
-			checker=Hash.new
-			checker['ip']=ip
-			checker['port']=port
-			checker['url']=url
-			checker['code']=code
-			checker['redirection']=loc
-			checker['md5']=fp
-			checker['server']=server
-			checker['timestamp']=timestamp
-			if Wmap::CidrTracker.new(:data_dir=>@data_dir).ip_trusted?(ip)
-				checker['status']="int_hosted"
-			else
-				checker['status']="ext_hosted"
-			end
-			return checker
-		rescue OpenSSL::SSL::SSLError => es  # handler to temporally hold the openssl bug in bay:  SSL_set_session: unable to find ssl method
-			checker=Hash.new
-			checker['ip']=ip
-			checker['port']=port
-			checker['url']=url
-			checker['code']=20000
-			checker['server']="Unknown SSL error: #{es}"
-			checker['md']=nil
-			checker['redirection']=nil
-			checker['timestamp']=timestamp
-			return checker
-		rescue Exception => ee
-			puts "Exception on method #{__method__} for #{url}: #{ee}" # if @verbose
-			return nil
+		puts "Checking out an unknown URL: #{url}" if @verbose
+		url=url.strip.downcase
+		raise "Invalid URL format: #{url}" unless is_url?(url)
+		timestamp=Time.now
+		host=url_2_host(url)
+		ip=host_2_ip(host)
+		port=url_2_port(url)
+		code=10000
+		if @url_code.key?(url)
+			code=@url_code[url]
+		else
+			code=response_code(url)
+		end
+		if code>=300 && code < 400
+			loc=landing_location(4,url)
+		else
+			loc=nil
+		end
+		if @url_finger_print.key?(url)
+			fp=@url_finger_print[url]
+		else
+			fp=response_body_md5(url)
+		end
+		if @url_server.key?(url)
+			server=@url_server[url]
+		else
+			server=get_server_header(url)
 		end
+		# save the data
+		checker=Hash.new
+		checker['ip']=ip
+		checker['port']=port
+		checker['url']=url
+		checker['code']=code
+		checker['redirection']=loc
+		checker['md5']=fp
+		checker['server']=server
+		checker['timestamp']=timestamp
+		if Wmap::CidrTracker.new(:data_dir=>@data_dir).ip_trusted?(ip)
+			checker['status']="int_hosted"
+		else
+			checker['status']="ext_hosted"
+		end
+		return checker
+	rescue OpenSSL::SSL::SSLError => es  # handler to temporally hold the openssl bug in bay:  SSL_set_session: unable to find ssl method
+		checker=Hash.new
+		checker['ip']=ip
+		checker['port']=port
+		checker['url']=url
+		checker['code']=20000
+		checker['server']="Unknown SSL error: #{es}"
+		checker['md']=nil
+		checker['redirection']=nil
+		checker['timestamp']=timestamp
+		return checker
+	rescue Exception => ee
+		puts "Exception on method #{__method__} for #{url}: #{ee}" # if @verbose
+		return nil
 	end
 	alias_method :check, :url_worker
 
 	# Parallel scanner - by utilizing fork manager 'parallel' to spawn numbers of child processes on multiple urls simultaneously
 	def url_workers (targets,num=@max_parallel)
-		begin
-			results=Array.new
-			targets -= ["", nil]
-			if targets.size > 0
-				puts "Start the url checker on the targets:\n #{targets}"
-				Parallel.map(targets, :in_processes => num) { |target|
-					url_worker(target)
-				}.each do |process|
-					if process.nil?
-						next
-					elsif process.empty?
-						#do nothing
-					else
-						results << process
-					end
+		results=Array.new
+		targets -= ["", nil]
+		if targets.size > 0
+			puts "Start the url checker on the targets:\n #{targets}"
+			Parallel.map(targets, :in_processes => num) { |target|
+				url_worker(target)
+			}.each do |process|
+				if process.nil?
+					next
+				elsif process.empty?
+					#do nothing
+				else
+					results << process
 				end
 			end
-			return results
-		rescue Exception => ee
-			puts "Exception on method #{__method__}: #{ee}" if @verbose
-			return nil
 		end
+		return results
+	rescue Exception => ee
+		puts "Exception on method #{__method__}: #{ee}" if @verbose
+		return nil
 	end
 	alias_method :checks, :url_workers
 
+=begin
 	# Test the URL and return the response code
 	def response_code (url)
 		puts "Check the http response code on the url: #{url}" if @verbose
-		response_code = 10000	# All unknown url connection exceptions go here
-		begin
-			raise "Invalid url: #{url}" unless is_url?(url)
-			url=url.strip.downcase
-			timeo = @http_timeout/1000.0
-			uri = URI.parse(url)
+		code = 10000	# All unknown url connection exceptions go here
+		raise "Invalid url: #{url}" unless is_url?(url)
+		url=url.strip.downcase
+		timeo = @http_timeout/1000.0
+		uri = URI.parse(url)
+		http = Net::HTTP.new(uri.host, uri.port)
+		http.open_timeout = timeo
+		http.read_timeout = timeo
+		if (url =~ /https\:/i)
+			http.use_ssl = true
+			#http.ssl_version = :SSLv3
+			# Bypass the remote web server cert validation test
+			http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+		end
+		request = Net::HTTP::Get.new(uri.request_uri)
+		response = http.request(request)
+		puts "Server response the following: #{response}" if @verbose
+		code = response.code.to_i
+		#response.finish if response.started?()
+		@url_code[url]=code
+		puts "Response code on #{url}: #{code}" if @verbose
+		return code
+	rescue Exception => ee
+		puts "Exception on method #{__method__} for #{url}: #{ee}" if @verbose
+		case ee
+		# rescue "Connection reset by peer" error type
+		when Errno::ECONNRESET
+			code=104
+		when Errno::ECONNABORTED,Errno::ETIMEDOUT
+			#code=10000
+		when Timeout::Error				# Quick fix
+			if (url =~ /https\:/i)		# try again for ssl timeout session, in case of default :TLSv1 failure
+				http.ssl_version = :SSLv3
+				response = http.request(request)
+				code = response.code.to_i
+				unless code.nil?
+					@ssl_version = http.ssl_version
+				end
+			end
+		else
+			#code=10000
+		end
+		@url_code[url]=code
+		return code
+	end
+
+	# Test the URL / site and return the redirection location (3xx response code only)
+	def redirect_location (url)
+		puts "Test the redirection location for the url: #{url}" if @verbose
+		location=""
+		raise "Invalid url: #{url}" unless is_url?(url)
+		url=url.strip.downcase
+		timeo = @http_timeout/1000.0
+		uri = URI.parse(url)
+		code = response_code (url)
+		if code >= 300 && code < 400
 			http = Net::HTTP.new(uri.host, uri.port)
 			http.open_timeout = timeo
 			http.read_timeout = timeo
 			if (url =~ /https\:/i)
 				http.use_ssl = true
-				#http.ssl_version = :SSLv3
 				# Bypass the remote web server cert validation test
 				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+				http.ssl_version = @ssl_version
 			end
 			request = Net::HTTP::Get.new(uri.request_uri)
 			response = http.request(request)
-			puts "Server response the following: #{response}" if @verbose
-			response_code = response.code.to_i
-			#response.finish if response.started?()
-			@url_code[url]=response_code
-			puts "Response code on #{url}: #{response_code}" if @verbose
-			return response_code
-		rescue Exception => ee
-			puts "Exception on method #{__method__} for #{url}: #{ee}" if @verbose
-			case ee
-				# rescue "Connection reset by peer" error type
-				when Errno::ECONNRESET
-					response_code=104
-				when Errno::ECONNABORTED,Errno::ETIMEDOUT
-					#response_code=10000
-				when Timeout::Error				# Quick fix
-					if (url =~ /https\:/i)		# try again for ssl timeout session, in case of default :TLSv1 failure
-						http.ssl_version = :SSLv3
-						response = http.request(request)
-						response_code = response.code.to_i
-						unless response_code.nil?
-							@ssl_version = http.ssl_version
-						end
-					end
-				else
-					#response_code=10000
+			puts "Response: #{response}" if @verbose
+			case response
+			when Net::HTTPRedirection then
+				location = response['location']
 			end
-			@url_code[url]=response_code
-			return response_code
-		end
-	end
-	alias_method :query, :response_code
-
-	# Test the URL / site and return the redirection location (3xx response code only)
-	def redirect_location (url)
-		begin
-			puts "Test the redirection location for the url: #{url}" if @verbose
-			location=""
-			raise "Invalid url: #{url}" unless is_url?(url)
-			url=url.strip.downcase
-			timeo = @http_timeout/1000.0
-			uri = URI.parse(url)
-			code = response_code (url)
-			if code >= 300 && code < 400
-				http = Net::HTTP.new(uri.host, uri.port)
-				http.open_timeout = timeo
-				http.read_timeout = timeo
-				if (url =~ /https\:/i)
-					http.use_ssl = true
-					# Bypass the remote web server cert validation test
-					http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-					http.ssl_version = @ssl_version
-				end
-				request = Net::HTTP::Get.new(uri.request_uri)
-				response = http.request(request)
-				puts "Response: #{response}" if @verbose
-				case response
-				when Net::HTTPRedirection then
-					location = response['location']
-				end
-			end
-			@url_redirection[url]=location
-			return location
-		rescue Exception => ee
-			puts "Exception on method redirect_location for URL #{url}: #{ee}" if @verbose
-			@url_redirection[url]=location
-			return location
 		end
+		@url_redirection[url]=location
+		return location
+	rescue Exception => ee
+		puts "Exception on method redirect_location for URL #{url}: #{ee}" if @verbose
+		return ""
 	end
 	alias_method :location, :redirect_location
 
 	# Test the URL / Site and return the landing url location (recursive with the depth = 4 )
-	def landing_location (depth=4, url)
-		begin
-			depth -= 1
-			return url if depth < 1
-			timeo = @http_timeout/1000.0
-			uri = URI.parse(url)
-			code = response_code (url)
-			if code >= 300 && code < 400
-				url = redirect_location (url)
-				url = landing_location(depth,url)
-			else
-				return url
-			end
+	def landing_location (depth=5, url)
+		depth -= 1
+		return url if depth < 1
+		timeo = @http_timeout/1000.0
+		uri = URI.parse(url)
+		code = response_code (url)
+		if code >= 300 && code < 400
+			url = redirect_location (url)
+			url = landing_location(depth,url)
+		else
 			return url
-		rescue Exception => ee
-			puts "Exception on method #{__method__} on URL #{url}: #{ee}" if @verbose
 		end
+		return url
+	rescue Exception => ee
+		puts "Exception on method #{__method__} on URL #{url}: #{ee}" if @verbose
 	end
+=end
 
 	# Test the URL / site and return the web server type from the HTTP header "server" field
 	def get_server_header (url)
-		begin
-			puts "Retrieve the server header field from the url: #{url}" if @verbose
-			server=String.new
-			raise "Invalid url: #{url}" unless is_url?(url)
-			url=url.strip.downcase
-			timeo = @http_timeout/1000.0
-			uri = URI.parse(url)
-			code = response_code (url)
-			http = Net::HTTP.new(uri.host, uri.port)
-			http.open_timeout = timeo
-			http.read_timeout = timeo
-			if (url =~ /https\:/i)
-				http.use_ssl = true
-				# Bypass the remote web server cert validation test
-				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-				http.ssl_version = @ssl_version
-			end
-			request = Net::HTTP::Get.new(uri.request_uri)
-			response = http.request(request)
-			server=response["server"]
-			server=server.gsub(/\,/,' ')
-			return server
-		rescue Exception => ee
-			puts "Exception on method get_server_header for URL #{url}: #{ee}" if @verbose
-			@url_server[url]=server
-			return server
+		puts "Retrieve the server header field from the url: #{url}" if @verbose
+		server=String.new
+		raise "Invalid url: #{url}" unless is_url?(url)
+		url=url.strip.downcase
+		timeo = @http_timeout/1000.0
+		uri = URI.parse(url)
+		code = response_code (url)
+		http = Net::HTTP.new(uri.host, uri.port)
+		http.open_timeout = timeo
+		http.read_timeout = timeo
+		if (url =~ /https\:/i)
+			http.use_ssl = true
+			# Bypass the remote web server cert validation test
+			http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+			http.ssl_version = @ssl_version
 		end
+		request = Net::HTTP::Get.new(uri.request_uri)
+		response = http.request(request)
+		server=response["server"]
+		server=server.gsub(/\,/,' ')
+		return server
+	rescue Exception => ee
+		puts "Exception on method get_server_header for URL #{url}: #{ee}" if @verbose
+		@url_server[url]=server
+		return server
 	end
 
 	# Use MD5 algorithm to fingerprint the URL / site response payload (web page content)
 	def response_body_md5(url)
 		puts "MD5 finger print page body content: #{url}" if @verbose
-		begin
-			raise "Invalid url: #{url}" unless is_url?(url)
-			url=url.strip.downcase
-			timeo = @http_timeout/1000.0
-			uri = URI.parse(url)
-			fp=""
-			http = Net::HTTP.new(uri.host, uri.port)
-			http.open_timeout = timeo
-			http.read_timeout = timeo
-			if (url =~ /https\:/i)
-				http.use_ssl = true
-				# Bypass the remote web server cert validation test
-				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-				http.ssl_version = @ssl_version
-			end
-			request = Net::HTTP::Get.new(uri.request_uri)
-			response = http.request(request)
-			response_body = response.body.to_s
-			fp=Digest::MD5.hexdigest(response_body) unless response_body.nil?
-			@url_finger_print[url] = fp
-			return fp
-		rescue Exception => ee
-			puts "Exception on method #{__method__}: #{ee}" if @verbose
+		raise "Invalid url: #{url}" unless is_url?(url)
+		url=url.strip.downcase
+		timeo = @http_timeout/1000.0
+		uri = URI.parse(url)
+		fp=""
+		http = Net::HTTP.new(uri.host, uri.port)
+		http.open_timeout = timeo
+		http.read_timeout = timeo
+		if (url =~ /https\:/i)
+			http.use_ssl = true
+			# Bypass the remote web server cert validation test
+			http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+			http.ssl_version = @ssl_version
 		end
+		request = Net::HTTP::Get.new(uri.request_uri)
+		response = http.request(request)
+		response_body = response.body.to_s
+		fp=Digest::MD5.hexdigest(response_body) unless response_body.nil?
+		@url_finger_print[url] = fp
+		return fp
+	rescue Exception => ee
+		puts "Exception on method #{__method__}: #{ee}" if @verbose
 	end
 	alias_method :md5, :response_body_md5
 
 	# Retrieve the remote web server certification, open it and return the cert content as a string
 	def get_certificate (url)
 		puts "Retrieve the remote web server SSL certificate in clear text: #{url}" if @verbose
-		begin
-			url=url.strip
-			raise "Invalid URL string: #{url}" unless is_ssl?(url)
-			client = HTTPClient.new
-			client.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
-			response = client.get(url)
-			cert = response.peer_cert
-			cer = OpenSSL::X509::Certificate.new(cert)
-			return cer.to_text
-		rescue Exception => ee
-			puts "Exception on method #{__method__} from #{url}: #{ee}"
-		end
+		url=url.strip
+		raise "Invalid URL string: #{url}" unless is_ssl?(url)
+		client = HTTPClient.new
+		client.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
+		response = client.get(url)
+		cert = response.peer_cert
+		cer = OpenSSL::X509::Certificate.new(cert)
+		return cer.to_text
+	rescue Exception => ee
+		puts "Exception on method #{__method__} from #{url}: #{ee}"
 		return nil
 	end
 	alias_method :get_cert, :get_certificate
@@ -315,19 +299,17 @@ class Wmap::UrlChecker
 	# Retrieve the X509 cert in the clear text from the remote web server, extract and return the common name field within the cert
 	def get_cert_cn (url)
 		puts "Extract the common name field from a X509 cert: #{cert}" if @verbose
-		begin
-			cert=get_certificate(url)
-			subject, cn = ""
-			if cert =~ /\n(.+)Subject\:(.+)\n/i
-				subject=$2
-			end
-			if subject =~/CN\=(.+)/i
-				cn=$1
-			end
-			return cn
-		rescue Exception => ee
-			puts "Error on method #{__method__} from #{cert}: #{ee}" if @verbose
+		cert=get_certificate(url)
+		subject, cn = ""
+		if cert =~ /\n(.+)Subject\:(.+)\n/i
+			subject=$2
+		end
+		if subject =~/CN\=(.+)/i
+			cn=$1
 		end
+		return cn
+	rescue Exception => ee
+		puts "Error on method #{__method__} from #{url}: #{ee}" if @verbose
 		return nil
 	end
 	alias_method :get_cn, :get_cert_cn