wmap 2.4.4

data/ruby_whois_patches/whois.registrygate.com.rb

@@ -0,0 +1,55 @@
+#--
+# Ruby Whois
+#
+# An intelligent pure Ruby WHOIS client and parser.
+#
+# Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
+#++
+
+require 'whois/record/parser/base'
+
+module Whois
+  class Record
+    class Parser
+
+      # Parser for the whois.registrygate.com server, added by Yang Li 02/10/2013.
+      class WhoisRegistrygateCom < Base
+
+		property_supported :admin_contacts do
+          build_contact("admin-", Whois::Record::Contact::TYPE_ADMIN)
+        end
+
+        property_supported :registrant_contacts do
+          build_contact("owner-", Whois::Record::Contact::TYPE_REGISTRANT)
+        end
+
+        property_supported :technical_contacts do
+          build_contact("tech-", Whois::Record::Contact::TYPE_TECHNICAL)
+        end
+
+        property_not_supported :billing_contacts 
+	
+      private
+
+        def build_contact(element, type)
+          reg=Record::Contact.new(:type => type)
+		  content_for_scanner.scan(/^(#{element}.*):\s+(.+)\n/).map do |entry|
+              reg["id"]=entry[1] if entry[0] =~ /id/i
+              reg["name"]=entry[1] if entry[0] =~ /name/i		  
+              reg["organization"]=entry[1] if entry[0]=~ /org/i
+              reg["address"]=entry[1] if entry[0]=~ /address/i
+              reg["city"]= entry[1] if entry[0]=~ /city/i
+              reg["zip"]=entry[1] if entry[0]=~ /pcode/i
+              reg["state"]=entry[1] if entry[0]=~ /state/i
+			  reg["country_code"]=entry[1] if entry[0]=~ /country/i
+			  reg["phone"]=entry[1] if entry[0]=~ /phone/i
+			  reg["fax"]=entry[1] if entry[0]=~ /fax/i
+			  reg["email"]=entry[1] if entry[0]=~ /email/i
+          end
+		  return reg
+        end	
+		
+      end
+    end
+  end
+end

data/ruby_whois_patches/whois.rrpproxy.net.rb

@@ -0,0 +1,61 @@
+#--
+# Ruby Whois
+#
+# An intelligent pure Ruby WHOIS client and parser.
+#
+# Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
+#++
+
+require 'whois/record/parser/base'
+
+module Whois
+  class Record
+    class Parser
+
+      # Parser for the whois.rrpproxy.net server, added by Yang Li 02/10/2013.
+      class WhoisRrpproxyNet < Base
+
+		property_supported :admin_contacts do
+          build_contact("admin-", Whois::Record::Contact::TYPE_ADMIN)
+        end
+
+        property_supported :registrant_contacts do
+          build_contact("owner-", Whois::Record::Contact::TYPE_REGISTRANT)
+        end
+
+        property_supported :technical_contacts do
+          build_contact("tech-", Whois::Record::Contact::TYPE_TECHNICAL)
+        end
+
+        property_not_supported :billing_contacts 
+	
+      private
+
+        def build_contact(element, type)
+          reg=Record::Contact.new(:type => type)
+		  content_for_scanner.scan(/^(#{element}.*):\s*(.+)\n/).map do |entry|
+              reg["id"]=entry[1] if entry[0] =~ /contact/i
+              if entry[0] =~ /(fname|lname)/i
+				if reg["name"].nil?
+					reg["name"]=entry[1]
+				else 
+					reg["name"]=reg["name"]+" "+entry[1]
+				end
+			  end			  
+              reg["organization"]=entry[1] if entry[0]=~ /organization/i
+              reg["address"]=entry[1] if entry[0]=~ /street/i
+              reg["city"]= entry[1] if entry[0]=~ /city/i
+              reg["zip"]=entry[1] if entry[0]=~ /zip/i
+              reg["state"]=entry[1] if entry[0]=~ /state/i
+			  reg["country_code"]=entry[1] if entry[0]=~ /country/i
+			  reg["phone"]=entry[1] if entry[0]=~ /phone/i
+			  reg["fax"]=entry[1] if entry[0]=~ /fax/i
+			  reg["email"]=entry[1] if entry[0]=~ /email/i
+          end
+		  return reg
+        end	
+		
+      end
+    end
+  end
+end

data/ruby_whois_patches/whois.sgnic.sg.rb

@@ -0,0 +1,130 @@
+#--
+# Ruby Whois
+#
+# An intelligent pure Ruby WHOIS client and parser.
+#
+# Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
+#++
+
+
+require 'whois/record/parser/base'
+
+
+module Whois
+  class Record
+    class Parser
+
+      #
+      # = whois.sgnic.sg parser
+      #
+      # Parser for the whois.sgnic.sg server.
+      #
+      # NOTE: This parser is just a stub and provides only a few basic methods
+      # to check for domain availability and get domain status.
+      # Please consider to contribute implementing missing methods.
+      # See WhoisNicIt parser for an explanation of all available methods
+      # and examples.
+      #
+      class WhoisSgnicSg < Base
+
+        property_supported :status do
+          content_for_scanner.scan(/^\s+Domain Status:\s+(.+?)\n/).flatten
+        end
+
+        property_supported :available? do
+          !!(content_for_scanner.strip == "Domain Not Found")
+        end
+
+        property_supported :registered? do
+          !available?
+        end
+
+
+        property_supported :created_on do
+          if content_for_scanner =~ /^\s+Creation Date:\s+(.*)\n/
+            Time.parse($1)
+          end
+        end
+
+        # This method is modified by Yang on 01/15/2013
+		property_supported :updated_on do
+          if content_for_scanner =~ /^\s+Modified Date:\s+(.*)\n/
+            Time.parse($1)
+          end	
+		end 
+
+        property_supported :expires_on do
+          if content_for_scanner =~ /^\s+Expiration Date:\s+(.*)\n/
+            Time.parse($1)
+          end
+        end
+
+        property_supported :nameservers do
+          if content_for_scanner =~ /Name Servers:\n((.+\n)+)\n/
+            values = case value = $1.downcase
+            # schema-1
+            when /^(?:\s+([\w.-]+)\n){2,}/
+              value.scan(/\s+([\w.-]+)\n/).map do |match|
+                { :name => match[0] }
+              end
+            when /^(?:\s+([\w.-]+)\s+\((.+)\)\n){2,}/
+              value.scan(/\s+([\w.-]+)\s+\((.+)\)\n/).map do |match|
+                { :name => match[0], :ipv4 => match[1] }
+              end
+            # schema-2
+            when /^(?:\s+([\w.-]+)){2,}/
+              value.strip.split(/\s+/).map do |name|
+                { :name => name }
+              end
+            else
+              Whois.bug!(ParserError, "Unknown nameservers format `#{value}'")
+            end
+
+            values.map do |params|
+              Record::Nameserver.new(params)
+            end
+          end
+        end
+		
+		# The following methods are implemented by Yang on 1/15/2013
+		# ----------------------------------------------------------------------------
+        property_supported :domain do
+          return $1 if content_for_scanner =~ /Domain Name:\s+(.*)\s+\n/i
+        end
+		
+		property_not_supported :domain_id
+		
+        property_supported :registrar do
+          return $1 if content_for_scanner =~ /Registrar:\s+(.*)\s+\n/i
+        end
+		
+        property_supported :admin_contacts do
+          build_contact("Administrative Contact", Whois::Record::Contact::TYPE_ADMIN)
+        end
+
+        property_supported :registrant_contacts do
+          build_contact("Registrant", Whois::Record::Contact::TYPE_REGISTRANT)
+        end
+
+        property_supported :technical_contacts do
+          build_contact("Technical Contact", Whois::Record::Contact::TYPE_TECHNICAL)
+        end
+		
+      private
+
+        def build_contact(element, type)
+          reg=Record::Contact.new(:type => type)
+		  values=$1 if content_for_scanner =~ /#{element}:\n\s*\n((.+\n)+)\n/
+		  values.scan(/^\s+(.+):\s*(.+)\s*\n/).map do |entry|
+              reg["name"]=entry[1] if entry[0] =~ /Name/i
+			  reg["email"]=entry[1] if entry[0]=~ /Email/i
+          end
+		  return reg
+        end	
+		# ----------------------------------------------------------------------------		
+
+      end
+
+    end
+  end
+end

data/ruby_whois_patches/whois.srs.net.nz.rb

@@ -0,0 +1,166 @@
+#--
+# Ruby Whois
+#
+# An intelligent pure Ruby WHOIS client and parser.
+#
+# Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
+#++
+
+
+require 'whois/record/parser/base'
+
+
+module Whois
+  class Record
+    class Parser
+
+      # Parser for the whois.srs.net.nz server.
+      #
+      # @note This parser is just a stub and provides only a few basic methods
+      #   to check for domain availability and get domain status.
+      #   Please consider to contribute implementing missing methods.
+      #
+      # @see Whois::Record::Parser::Example
+      #   The Example parser for the list of all available methods.
+      #
+      class WhoisSrsNetNz < Base
+
+        # @see http://dnc.org.nz/content/srs-whois-spec-1.0.html
+        property_supported :status do
+          if content_for_scanner =~ /query_status:\s(.+)\n/
+            case (s = $1.downcase)
+            when "200 active"
+              :registered
+            # The domain is no longer active but is in the period prior
+            # to being released for general registrations
+            when "210 pendingrelease"
+              :redemption
+            when "220 available"
+              :available
+            when "404 request denied"
+              :error
+            when /invalid characters/
+              :invalid
+            else
+              Whois.bug!(ParserError, "Unknown status `#{s}'.")
+            end
+          else
+            Whois.bug!(ParserError, "Unable to parse status.")
+          end
+        end
+
+        property_supported :available? do
+          status == :available
+        end
+
+        property_supported :registered? do
+          status == :registered || status == :redemption
+        end
+
+
+        property_supported :created_on do
+          if content_for_scanner =~ /domain_dateregistered:\s(.+)\n/
+            Time.parse($1)
+          end
+        end
+
+        property_supported :updated_on do
+          if content_for_scanner =~ /domain_datelastmodified:\s(.+)\n/
+            Time.parse($1)
+          end
+        end
+
+        property_supported :expires_on do
+          if content_for_scanner =~ /domain_datebilleduntil:\s(.+)\n/
+            Time.parse($1)
+          end
+        end
+
+        property_supported :nameservers do
+          content_for_scanner.scan(/ns_name_[\d]+:\s(.+)\n/).flatten.map do |name|
+            Record::Nameserver.new(:name => name)
+          end
+        end
+
+        # Checks whether the response has been throttled.
+        #
+        # @return [Boolean]
+        #
+        # @example
+        #   query_status: 440 Request Denied
+        #
+        def response_throttled?
+          cached_properties_fetch(:response_throttled?) do
+            !!(content_for_scanner =~ /^query_status: 440 Request Denied/)
+          end
+        end
+
+        # NEWPROPERTY
+        def valid?
+          cached_properties_fetch(:valid?) do
+            !invalid?
+          end
+        end
+
+        # NEWPROPERTY
+        def invalid?
+          cached_properties_fetch(:invalid?) do
+            status == :invalid
+          end
+        end
+
+		# The following methods are implemented by Yang Li on 01/24/2013
+		# ----------------------------------------------------------------------------
+        property_supported :domain do
+          return $1 if content_for_scanner =~ /domain_name:\s+(.*)\n/i
+        end
+		
+		property_not_supported :domain_id 
+		
+        property_supported :registrar do
+          reg=Record::Registrar.new
+		  content_for_scanner.scan(/^registrar_(.+):\s+(.+)\n/).map do |entry|
+			reg["name"] = entry[1] if entry[0] =~ /name/i
+			reg["organization"] = entry[1] if entry[0] =~ /name/i
+			reg["id"] = entry[1] if entry[0] =~ /phone/i
+			reg["url"] = entry[1] if entry[0] =~ /email/i			
+          end
+		  return reg
+        end
+
+        property_supported :registrant_contacts do
+          build_contact("registrant_", Whois::Record::Contact::TYPE_REGISTRANT)
+        end
+
+        property_supported :admin_contacts do
+          build_contact("admin_", Whois::Record::Contact::TYPE_ADMIN)
+        end
+
+        property_supported :technical_contacts do
+          build_contact("technical_", Whois::Record::Contact::TYPE_TECHNICAL)
+        end
+
+        property_not_supported :billing_contacts 
+		
+      private
+
+        def build_contact(element, type)
+          reg=Record::Contact.new(:type => type)
+		  content_for_scanner.scan(/^#{element}(.+):\s+(.+)\n/).map do |entry|
+              reg["name"]=entry[1] if entry[0] =~ /name/i
+              reg["organization"]=entry[1] if entry[0]=~ /name/i
+              reg["address"]=entry[1] if entry[0]=~ /address1/i
+              reg["city"]= entry[1] if entry[0]=~ /city/i
+			  reg["country"]=entry[1] if entry[0]=~ /country/i
+			  reg["phone"]=entry[1] if entry[0]=~ /phone/i
+			  reg["fax"]=entry[1] if entry[0]=~ /fax/i
+			  reg["email"]=entry[1] if entry[0]=~ /email/i
+          end
+		  return reg
+        end	
+		# ----------------------------------------------------------------------------
+		
+      end
+    end
+  end
+end

data/ruby_whois_patches/whois.tucows.com.rb

@@ -0,0 +1,70 @@
+#--
+# Ruby Whois
+#
+# An intelligent pure Ruby WHOIS client and parser.
+#
+# Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
+#++
+
+require 'whois/record/parser/base'
+
+module Whois
+  class Record
+    class Parser
+
+      # Parser for the whois.tucows.com server, added by Yang Li 02/10/2013.
+      class WhoisTucowsCom < Base
+
+        property_supported :registrar do
+          reg=Record::Registrar.new
+		  if content_for_scanner =~ /Registration Service Provider:\n((.+\n)+)\n/
+			line_num=1
+			$1.split(%r{\n}).each do |entry|
+				reg["name"] = entry.strip.split(',')[0] if line_num==1
+				reg["organization"] = entry.strip.split(',')[0] if line_num==1
+				reg["url"] = entry.strip.split(',')[1] if line_num==1		
+				line_num=line_num+1
+			end
+          end
+		  return reg
+        end
+		
+		property_supported :admin_contacts do
+          build_contact("Administrative Contact", Whois::Record::Contact::TYPE_ADMIN)
+        end
+
+        property_supported :registrant_contacts do
+          build_contact("Registrant", Whois::Record::Contact::TYPE_REGISTRANT)
+        end
+
+        property_supported :technical_contacts do
+          build_contact("Technical Contact", Whois::Record::Contact::TYPE_TECHNICAL)
+        end
+
+        property_not_supported :billing_contacts 
+	
+      private
+
+        def build_contact(element, type)
+          reg=Record::Contact.new(:type => type)
+		  if content_for_scanner =~ /#{element}:\n((.+\n)+)\n/i
+			  line_num=1
+			  addrs=''
+			  $1.split(%r{\n}).each do |entry|
+				  reg["name"]=entry.strip if line_num==1
+				  reg["organization"]=entry.strip if line_num==1
+				  reg["address"]=entry.strip if line_num==2
+				  reg["phone"]=$1 if entry =~ /(\d{7,})/i
+				  reg["fax"]=entry.strip.split('Fax:').last.strip if entry =~ /Fax/i
+				  reg["email"]=$1 if entry =~ /(\w+\@.+\.\w+)/i
+				  line_num=line_num+1
+			  end
+			end
+			return reg
+        end	
+		# ----------------------------------------------------------------------------
+		
+      end
+    end
+  end
+end

data/ruby_whois_patches/whois.twnic.net.tw.rb

@@ -0,0 +1,133 @@
+#--
+# Ruby Whois
+#
+# An intelligent pure Ruby WHOIS client and parser.
+#
+# Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
+#++
+
+
+require 'whois/record/parser/base'
+
+
+module Whois
+  class Record
+    class Parser
+
+      #
+      # = whois.twnic.net.tw
+      #
+      # Parser for the whois.twnic.net.tw server.
+      #
+      # NOTE: This parser is just a stub and provides only a few basic methods
+      # to check for domain availability and get domain status.
+      # Please consider to contribute implementing missing methods.
+      # See WhoisNicIt parser for an explanation of all available methods
+      # and examples.
+      #
+      class WhoisTwnicNetTw < Base
+
+        property_supported :status do
+          if available?
+            :available
+          else
+            :registered
+          end
+        end
+
+        property_supported :available? do
+          !!(content_for_scanner.strip == "No Found")
+        end
+
+        property_supported :registered? do
+          !available?
+        end
+
+
+        property_supported :created_on do
+          if content_for_scanner =~ /Record created on ([^ ]+) .+\n/
+            Time.parse($1)
+          end
+        end
+
+        property_not_supported :updated_on
+
+        property_supported :expires_on do
+          if content_for_scanner =~ /Record expires on ([^ ]+) .+\n/
+            Time.parse($1)
+          end
+        end
+
+        property_supported :nameservers do
+          if content_for_scanner =~ /Domain servers in listed order:\n((.+\n)+)\n/
+            $1.split("\n").map do |name|
+              Record::Nameserver.new(:name => name.strip)
+            end
+          end
+        end
+
+		# The following methods are implemented by Yang Li on 01/28/2013
+		# ----------------------------------------------------------------------------
+        property_supported :domain do
+          return $1 if content_for_scanner =~ /Domain Name:\s+(.*)\n/i
+        end
+		
+		property_not_supported :domain_id
+		
+        property_not_supported :registrar 
+		
+		property_not_supported :billing_contacts
+
+        property_supported :registrant_contacts do
+          build_contact("Registrant", Whois::Record::Contact::TYPE_REGISTRANT)
+        end
+
+        property_supported :technical_contacts do
+          build_contact_1("Technical Contact", Whois::Record::Contact::TYPE_TECHNICAL)
+        end
+
+        property_supported :admin_contacts do
+          build_contact_1("Administrative Contact", Whois::Record::Contact::TYPE_ADMIN)
+        end
+		
+      private
+
+        def build_contact(element, type)
+          reg=Record::Contact.new(:type => type)
+		  if content_for_scanner =~ /#{element}:\n((.+\n)+)\n/i
+			  line_num=1
+			  $1.split(%r{\n}).each do |line|
+				reg["organization"]=line.strip if line_num==1
+				reg["name"]=line.split(%r{\s{2,}})[1] if line_num==2
+				reg["email"]=line.split(%r{\s{2,}})[2] if line_num==2
+				reg["phone"]=line.strip if line_num==3
+				reg["fax"]=line.strip if line_num==4
+				reg["address"]=line.strip if line_num==5
+				reg["city"]=line.strip if line_num==6
+				reg["country_code"]=line.strip if line_num==7
+				line_num=line_num+1
+			  end			  
+          end
+		  return reg
+        end	
+		
+        def build_contact_1(element, type)
+          reg=Record::Contact.new(:type => type)
+		  if content_for_scanner =~ /#{element}:\n((.+\n)+)\n/i
+			  line_num=1
+			  $1.split(%r{\n}).each do |line|
+				reg["name"]=line.split(%r{\s{2,}})[1] if line_num==1
+				reg["email"]=line.split(%r{\s{2,}})[2] if line_num==1
+				reg["phone"]=line.strip if line_num==2
+				reg["fax"]=line.strip if line_num==3
+				line_num=line_num+1
+			  end			  
+          end
+		  return reg
+        end	
+		# ----------------------------------------------------------------------------		
+
+      end
+    end
+  end
+end

data/settings/discovery_ports

@@ -0,0 +1,24 @@
+# Default Common TCP ports for HTTP(S) service discovery
+# Used by the Wmap::PortScanner class
+80
+81
+443
+591
+901
+2086
+2087
+2095
+2096
+3128
+4443
+8000
+8001
+8080
+8118
+8443
+8888
+9000
+9090
+9443
+9999
+20720

data/settings/google_keywords.txt

@@ -0,0 +1,9 @@
+# Default list of Google search keywords (known company legal names, for example) for the service discovery, one entry at per line
+# Used by the Wmap::GoogleSearchScraper class
+
+# Official whole name
+Google Inc.
+
+# 2014 purchase
+DeepMind Technologies
+

data/settings/google_locator.txt

@@ -0,0 +1,23 @@
+# Default list of all major Google search engines for our google discovery, in one entry per line format
+# To be used by the Wmap::GoogleSearchScraper class
+# Refer to: http://seobabes.blog.com/seo/list-of-all-google-search-engines-in-different-country/ for more details
+
+http://www.google.ca/
+http://www.google.ch/
+http://www.google.co.in/
+http://www.google.co.jp/
+http://www.google.co.kr/
+http://www.google.com/
+http://www.google.com.au/
+http://www.google.com.hk/
+http://www.google.co.nz/
+http://www.google.co.uk/
+http://www.google.co.za/
+http://www.google.de/
+http://www.google.es/
+http://www.google.fr/
+http://www.google.ie/
+http://www.google.lt/
+http://www.google.nl/
+http://www.google.ru/
+http://www.google.se/