domain-probe 1.2.0 → 2.0.0

data/lib/domain-probe.rb

@@ -1,15 +1,15 @@
+require 'domain-probe/tld'
 require 'domain-probe/resolver_patch'
 require 'domain-probe/util'
 require 'domain-probe/policy'
 require 'domain-probe/guessing_policy'
 require 'domain-probe/crawling_policy'
+require 'domain-probe/thread_executor'
 require 'set'
 
 module Probe
   
   class DomainProbe
-    
-     MAX_RECURSIVE_QUERY_DEPTH = 6
      
      @policies = []
      
@@ -32,213 +32,128 @@ module Probe
      # collect the records under domain domain
      #
      def self.collect_records domain
-       self.new(domain).collect_records
+       self.new().collect_records domain
      end
      
      #
      # collect the ns records
      #
      def self.collect_ns_records domain
-       self.new(domain).collect_ns_records
+       self.new().collect_ns_records domain
      end
-     
-     #
-     # ----------------------------------------
-     #
-     #
-     #
-     def initialize(domain)
-       @domain = domain.downcase
-       @a_records     = []
-       @aaaa_records  = []
-       @cname_records = []
-       @mx_records    = []
-       @mr_records    = []
-       @txt_records   = []
-       @hinfo_records = []
-       @srv_records   = []
-       @spf_records    = []
-       @ns_records    = []
-       @semaphore     = Mutex.new
-     end
-     
-     attr_accessor :a_records
-     attr_accessor :aaaa_records
-     attr_accessor :cname_records
-     attr_accessor :mx_records
-     attr_accessor :mr_records
-     attr_accessor :txt_records
-     attr_accessor :hinfo_records
-     attr_accessor :srv_records
-     attr_accessor :ns_records
-     attr_accessor :spf_records
-     attr_accessor :semaphore
-     attr_accessor :domain
       
       #
       # collect the records under domain domain
+      # IF zone detection fails, return nil?
       #
-      def collect_records
-          raise ArgumentError, "Domain name is expected" unless self.domain
-          domain = zone = Util.detect_zone(self.domain)
-          return [] if !zone
-          
-          threads = []
-          nameservers = Util.detect_nameservers zone
+      def collect_records domain
+          raise ArgumentError, "Domain name is expected" unless domain
+          domain = zone = Util.detect_zone(domain.downcase)
+          return {} if !zone || is_tld(zone)
+        
+          nameservers = detect_nameservers zone
           possible_domains = Set.new
           
           DomainProbe.policies.each{ |policy|
             possible_domains |= policy.possible_host_under_domain(zone)
           }
+
+          executor = ThreadExecutor.new
        
-          #A/CNAME records
+          #check to see if a wildcard record there? if yes, stop the probing
+          executor.submit_task(wildcard_domain_name(zone),Net::DNS::A,zone,nameservers)
+          executor.submit_task(wildcard_domain_name(zone),Net::DNS::AAAA,zone,nameservers)
+          executor.execute_util_finished
+          
+          #start again
+          executor.start_threads
+          
+          #A/AAAA/CNAME records
           possible_domains.each{ |possible_host|
-               possible_domain = possible_host + "." + zone
-
-               threads << threaded_resolve(possible_domain,Net::DNS::A,zone,nameservers) 
-               #AAAA
-               threads << threaded_resolve(possible_domain,Net::DNS::AAAA,zone,nameservers)  
-          }
+               executor.submit_task(possible_domain_name(possible_host,zone),Net::DNS::A,zone,nameservers)
+               executor.submit_task(possible_domain_name(possible_host,zone),Net::DNS::AAAA,zone,nameservers)
+          } unless !executor.a_records.empty? || !executor.aaaa_records.empty? || !executor.cname_records.empty?
           
           #A/CNAME records
-          threads << threaded_resolve(domain,Net::DNS::A,zone,nameservers) 
+          executor.submit_task(domain,Net::DNS::A,zone,nameservers) 
           #AAAA
-          threads << threaded_resolve(domain,Net::DNS::AAAA,zone,nameservers)
+          executor.submit_task(domain,Net::DNS::AAAA,zone,nameservers)
           
           #MX records
-          threads << threaded_resolve(domain,Net::DNS::MX,zone,nameservers) 
+          executor.submit_task(domain,Net::DNS::MX,zone,nameservers) 
           
           #MR records
-          threads << threaded_resolve(domain,Net::DNS::MR,zone,nameservers)
+          executor.submit_task(domain,Net::DNS::MR,zone,nameservers)
           
           #TXT records
-          threads << threaded_resolve(domain,Net::DNS::TXT,zone,nameservers)
+          executor.submit_task(domain,Net::DNS::TXT,zone,nameservers)
           
           #HINFO records
-          threads << threaded_resolve(domain,Net::DNS::HINFO,zone,nameservers)
+          executor.submit_task(domain,Net::DNS::HINFO,zone,nameservers)
           
           #SRV records
-          threads << threaded_resolve(domain,Net::DNS::SRV,zone,nameservers)
+          executor.submit_task(domain,Net::DNS::SRV,zone,nameservers)
 
           #SPF records
-          threads << threaded_resolve(domain,Net::DNS::SPF,zone,nameservers)
+          executor.submit_task(domain,Net::DNS::SPF,zone,nameservers)
+          
+          #SOA records
+          executor.submit_task(domain,Net::DNS::SOA,zone,nameservers)
           
           #NS records
-          threads << threaded_resolve(domain,Net::DNS::NS,zone,nameservers)          
-
-          threads.each do |t|
-            t.join
-          end
+          executor.submit_task(domain,Net::DNS::NS,zone,nameservers)          
 
+          executor.execute_util_finished
+    
           results = {
-            "a" => self.a_records,
-            "aaaa" => self.aaaa_records,
-            "cname" => self.cname_records,
-            "mx" => self.mx_records,
-            "mr" => self.mr_records,
-            "txt" => self.txt_records,
-            "hinfo" => self.hinfo_records,
-            "srv" => self.srv_records,
-            "ns" => self.ns_records,
-            "spf" => self.spf_records
+            "a" => executor.a_records,
+            "aaaa" => executor.aaaa_records,
+            "cname" => executor.cname_records,
+            "mx" => executor.mx_records,
+            "mr" => executor.mr_records,
+            "txt" => executor.txt_records,
+            "hinfo" => executor.hinfo_records,
+            "srv" => executor.srv_records,
+            "ns" => executor.ns_records.select{|record| record[:name] != "@"},
+            "spf" => executor.spf_records,
+            "soa" => executor.soa_records,
+            "zone" => Util.remove_trailing_dot_if_any(zone)
           }          
       end
       
-      def collect_ns_records
-         raise ArgumentError, "Domain name is expected" unless self.domain
-         domain = zone = Util.detect_zone(self.domain)
-         return [] if !zone
+      def collect_ns_records domain
+         raise ArgumentError, "Domain name is expected" unless domain
+         domain = zone = Util.detect_zone(domain.downcase)
+         return [] if !zone || is_tld(zone)
           
-         threads = []
-         nameservers = Util.detect_nameservers zone
+         executor = ThreadExecutor.new
+         nameservers = Util.detect_nameservers zone.split(".").last
            
          #NS records
-         threads << threaded_resolve(domain,Net::DNS::NS,zone,nameservers)          
-
-         threads.each do |t|
-           t.join
-         end
-            
-        self.ns_records
+         executor.submit_task(domain,Net::DNS::NS,zone,nameservers)
+         executor.execute_util_finished
+         executor.ns_records
       end
-       
-    private
-
-      #
-      # the method will be called under the threaded execution context
-      # Attention: 
-      # 1. tail recursive call happened when we get an un-authorized reply
-      # 2. depth limit to avoid infinite recursion
-      #
-      def resolve_recursive(domain,type,zone,nameservers,depth)
-        
-        depth += 1
-        return if depth > MAX_RECURSIVE_QUERY_DEPTH
-        single_record_handler = lambda{  |record|
-    
-          if record.name && (record.name == Util.append_trailing_dot_if_necessary(zone) || Util.is_subdomain?(record.name,zone))
-           #A little overhead here
-           self.semaphore.synchronize {
-             case record.type
-             when "A"      
-               self.a_records << record.to_hash(zone) unless self.a_records.include? record.to_hash(zone)
-             when "AAAA"
-               self.aaaa_records << record.to_hash(zone) unless self.aaaa_records.include? record.to_hash(zone)
-             when "CNAME"
-               self.cname_records << record.to_hash(zone) unless self.cname_records.include? record.to_hash(zone)               
-             when "NS"
-               self.ns_records << record.to_hash(zone) unless self.ns_records.include? record.to_hash(zone)
-             when "MX"
-               self.mx_records << record.to_hash(zone) unless self.mx_records.include? record.to_hash(zone)
-             when "MR"
-               self.mr_records << record.to_hash(zone) unless self.mr_records.include? record.to_hash(zone) 
-             when "TXT"
-               self.txt_records << record.to_hash(zone) unless self.txt_records.include? record.to_hash(zone)
-             when "SRV"
-               self.srv_records << record.to_hash(zone) unless self.srv_records.include? record.to_hash(zone)
-             when "HINFO"
-               self.hinfo_records << record.to_hash(zone) unless self.hinfo_records.include? record.to_hash(zone) 
-             when "SPF"
-                self.spf_records << record.to_hash(zone) unless self.spf_records.include? record.to_hash(zone)  
-             else
-               #nothing to do?
-             end
-           } 
-         end
-       }
-     
-       #
-       begin
-         
-         resolver = nameservers.empty? ? Net::DNS::Resolver.new : Net::DNS::Resolver.new(:nameserver => nameservers)
-         resolver.log_level = Net::DNS::ERROR
-         packet = resolver.query(domain, type)
-         return unless packet
-         
-         if packet.header.auth? || !packet.answer.empty?
-            packet.answer.each &single_record_handler if packet.answer
-            packet.authority.each &single_record_handler if packet.authority
-            packet.additional.each &single_record_handler if packet.additional
-         elsif !packet.authority.empty?
-            #following down
-            nameservers = Util.organize_authorities(packet.authority,packet.additional)
-            #tail recursive
-            resolve_recursive(domain,type,zone,nameservers,depth)
-         end
-         
-       rescue =>ex
-         puts "#{ex.class}: #{ex.message}"
-       end
+      
+      private
+      
+      def detect_nameservers domain
+        nameserver_names = []
+        collect_ns_records(domain).each{ |authority|
+          nameserver_names << authority[:nsdname]
+        }
+        Util.resolve_addresses(nameserver_names)
       end
-
+      
+      def possible_domain_name(host,zone)
+          host + "." + zone
+      end
+      
       #
-      # The thread should benifit us, IF NOT, disable it
+      # used to detect wildcard records
       #
-      def threaded_resolve(domain,type,zone,nameservers)
-          Thread.new(domain,type,zone,nameservers) { |domain,type,zone,nameservers|
-              depth = 0
-              resolve_recursive(domain,type,zone,nameservers,depth)
-          }
+      def wildcard_domain_name(zone)
+          "*." + zone
       end
       
       register_policy GuessingPolicy.new

data/lib/domain-probe/crawling_policy.rb

@@ -1,8 +1,6 @@
 require 'set'
 require 'nokogiri'
 require 'open-uri'
-require 'domain-probe/util'
-require 'domain-probe/policy'
 
 module Probe
   

data/lib/domain-probe/guessing_policy.rb

@@ -1,4 +1,3 @@
-require 'domain-probe/policy'
 module Probe
   
   #
@@ -36,6 +35,7 @@ module Probe
     dir
     direct
     discovery
+    ditu
     docs
     download
     downloads
@@ -63,6 +63,7 @@ module Probe
     love
     m
     mail
+    map
     maps
     media
     message
@@ -87,6 +88,7 @@ module Probe
     resource
     resources
     sg
+    shop
     shopping
     sms
     smtp

data/lib/domain-probe/resolver_patch.rb

@@ -7,6 +7,34 @@ require 'net/dns/resolver'
 #
 module Net 
   module DNS
+    
+    #to support *
+    class Resolver
+      
+      def valid?(name)
+        if name =~ /[^-\w\.\*]/
+          raise ArgumentError, "Invalid domain name #{name}"
+        else
+          true
+        end
+      end
+    end
+    
+    #to support *
+    class Question
+    
+      def check_name(name)
+        name.strip!
+        if name =~ /[^\w\.\-_\*]/
+          raise NameError, "Question name #{name.inspect} not valid"          
+        else
+          name
+        end
+      rescue
+        raise NameError, "Question name #{name.inspect} not valid"                  
+      end
+    end
+    
     class RR
       
       #
@@ -31,8 +59,19 @@ module Net
         
         {:name => cooked_value, :ttl => ttl}
       end
-      
-  
+    
+      class SOA
+        def to_hash zone
+          super(zone).merge({ :mname => self.mname.to_s,
+                              :rname => self.rname.to_s,
+                              :serial => self.serial.to_s,
+                              :refresh => self.refresh.to_s,
+                              :retry => self.retry.to_s,
+                              :expire => self.expire.to_s,
+                              :minimum => self.minimum.to_s,
+                              })
+        end
+      end
       
       class A
         def to_hash zone

data/lib/domain-probe/thread_executor.rb

@@ -0,0 +1,168 @@
+require 'thread'
+
+module Probe
+  
+  class ThreadExecutor
+    
+    MAX_RECURSIVE_QUERY_DEPTH = 6
+     #
+     # ----------------------------------------
+     #
+     #
+     #
+     def initialize(size = 10)
+       @a_records     = []
+       @aaaa_records  = []
+       @cname_records = []
+       @mx_records    = []
+       @mr_records    = []
+       @txt_records   = []
+       @hinfo_records = []
+       @srv_records   = []
+       @spf_records   = []
+       @soa_records   = []
+       @ns_records    = []
+       @mutex         = Mutex.new
+       @jobs          = Queue.new
+       @workers       = start_threads(size)
+     end
+     
+     attr_accessor :a_records
+     attr_accessor :aaaa_records
+     attr_accessor :cname_records
+     attr_accessor :mx_records
+     attr_accessor :mr_records
+     attr_accessor :txt_records
+     attr_accessor :hinfo_records
+     attr_accessor :srv_records
+     attr_accessor :ns_records
+     attr_accessor :spf_records
+     attr_accessor :soa_records
+     attr_accessor :mutex
+     attr_accessor :domain
+     attr_accessor :jobs
+     attr_accessor :workers
+     
+     def start_threads(size = 10)
+      
+      raise "Live threads are still there" if self.workers && !self.workers.select{|t| t.status}.empty?
+       
+      self.workers       = Array.new(size) do |i|
+          Thread.new do
+            catch(:done) do
+              loop do
+                  job, args = self.jobs.pop
+                  job.call(*args)
+              end
+            end
+          end
+        end
+     end
+
+     #
+     #
+     def execute_util_finished
+       
+        self.workers.length.times do
+          post_task { throw :done }
+        end
+        
+        self.workers.map(&:join)
+     end
+
+     #
+     #
+     def post_task(*args, &block)
+       @jobs << [block, args] if block_given?
+     end
+     
+     def post_result record,zone
+       
+       return unless record && record.kind_of?(Net::DNS::RR)
+        #A little overhead here
+        self.mutex.synchronize {
+          record_hash = record.to_hash(zone) 
+          case record.type
+          when "A" 
+            self.a_records << record_hash unless self.a_records.include? record_hash
+          when "AAAA"
+            self.aaaa_records << record_hash unless self.aaaa_records.include? record_hash
+          when "CNAME"
+            self.cname_records << record_hash unless self.cname_records.include? record_hash               
+          when "NS"
+            self.ns_records << record_hash unless self.ns_records.include? record_hash
+          when "MX"
+            self.mx_records << record_hash unless self.mx_records.include? record_hash
+          when "MR"
+            self.mr_records << record_hash unless self.mr_records.include? record_hash 
+          when "TXT"
+            self.txt_records << record_hash unless self.txt_records.include? record_hash
+          when "SRV"
+            self.srv_records << record_hash unless self.srv_records.include? record_hash
+          when "HINFO"
+            self.hinfo_records << record_hash unless self.hinfo_records.include? record_hash 
+          when "SPF"
+            self.spf_records << record_hash unless self.spf_records.include? record_hash 
+          when "SOA"
+            self.soa_records << record_hash unless self.soa_records.include? record_hash 
+          else
+            #nothing to do?
+          end
+        }
+     end
+     
+     #
+     #
+     def submit_task(domain,type,zone,nameservers)
+         post_task(domain,type,zone,nameservers) { |domain,type,zone,nameservers|
+             depth = 0
+             resolve_recursive(domain,type,zone,nameservers,depth)
+         }
+     end
+     
+     private
+
+       #
+       # the method will be called under the threaded execution context
+       # Attention: 
+       # 1. tail recursive call happened when we get an un-authorized reply
+       # 2. depth limit to avoid infinite recursion
+       #
+       def resolve_recursive(domain,type,zone,nameservers,depth)
+
+         return if nameservers.empty?
+         depth += 1
+         return if depth > MAX_RECURSIVE_QUERY_DEPTH
+         single_record_handler = lambda{  |record|
+
+           if record.name && (record.name == Util.append_trailing_dot_if_necessary(zone) || Util.is_subdomain?(record.name,zone))
+             post_result record,zone
+           end
+         }
+
+        #
+        begin
+
+          resolver = Net::DNS::Resolver.new(:nameserver => nameservers,:udp_timeout => 2)
+          resolver.log_level = Net::DNS::ERROR
+          packet = resolver.query(domain, type)
+          return unless packet && !packet.header.error?
+
+          if packet.header.auth? || !packet.answer.empty?
+             packet.answer.each &single_record_handler if packet.answer
+             packet.authority.each &single_record_handler if packet.authority
+             packet.additional.each &single_record_handler if packet.additional
+          elsif !packet.authority.empty?
+             #following down
+             nameservers = Util.organize_authorities(packet.authority,packet.additional)
+             #tail recursive
+             resolve_recursive(domain,type,zone,nameservers,depth)
+          end
+
+        rescue =>ex
+          puts "#{ex.class} happened: #{ex.message} while resolving #{domain} of type #{type} at nameserves #{nameservers.join(',')}\n"
+        end
+       end
+       
+  end
+end

data/lib/domain-probe/util.rb

@@ -1,7 +1,3 @@
-require 'domain-probe/resolver_patch'
-#
-# 
-#
 module Probe
   
   #
@@ -21,19 +17,20 @@ module Probe
       zone = nil
       begin
         names.size.times { |index|
-          resovler = Net::DNS::Resolver.new
-          packet = resovler.query(domain, Net::DNS::SOA)
+          resolver = Net::DNS::Resolver.new(:udp_timeout => 2)
+          resolver.log_level = Net::DNS::ERROR
+          packet = resolver.query(domain, Net::DNS::SOA)
           packet.answer.each { |record|
               next unless record.name == domain && record.type == "SOA"
               zone = domain
               break
-          } if packet && packet.answer
+          } if packet && !packet.header.error? && packet.answer
         
           return remove_trailing_dot_if_any(zone) unless !zone
           domain = domain.slice(names[index].length + 1,domain.length - names[index].length - 1)
         } 
       rescue  => ex
-        puts "#{ex.class}: #{ex.message}"
+        puts "#{ex.class} happened: #{ex.message} while detecting zone of #{domain}\n"
       end
       
       zone
@@ -49,21 +46,22 @@ module Probe
       nameserver_ips = []
       nameserver_names = []
       begin
-        resovler = Net::DNS::Resolver.new
-        packet = resovler.query(zone, Net::DNS::NS)
+        resolver = Net::DNS::Resolver.new(:udp_timeout => 2)
+        resolver.log_level = Net::DNS::ERROR
+        packet = resolver.query(zone, Net::DNS::NS)
         packet.answer.each { |record|
             next unless record.name == append_trailing_dot_if_necessary(zone) && record.type == "NS"
             nameserver_names << record.nsdname
             packet.additional.select{ |x| x.name == record.nsdname && x.type == "A" }.each{ |rr|
                 nameserver_ips << rr.address.to_s
             }
-        } if packet && packet.answer
+        } if packet && !packet.header.error? && packet.answer
         
         if nameserver_ips.empty? 
           nameserver_ips += resolve_addresses(nameserver_names)
         end
       rescue  => ex
-        puts "#{ex.class}: #{ex.message}"
+        puts "#{ex.class} happened: #{ex.message} while detecting nameservers of #{zone}\n"
       end
       
       nameserver_ips
@@ -128,8 +126,6 @@ module Probe
       domain.end_with?(zone_with_trailing_dot) && domain[domain.index(zone_with_trailing_dot) - 1,1] == "."
     end
     
-    private
-    
     #
     # resolve the names to ip addresses
     #
@@ -138,16 +134,18 @@ module Probe
       return [] unless names && !names.empty?
       addresses = []
       begin
-          resovler = Net::DNS::Resolver.new
+          resolver = Net::DNS::Resolver.new(:udp_timeout => 2)
+          resolver.log_level = Net::DNS::ERROR
           names.each{ |name|
-              packet = resovler.query(name, Net::DNS::A)
+              packet = resolver.query(name, Net::DNS::A)
+              next unless packet && !packet.header.error?
               packet.answer.each { |record|
                   next unless record.type == "A"
                   addresses << record.address.to_s
               }
           }
       rescue  => ex
-        puts "#{ex.class}: #{ex.message}"
+        puts "#{ex.class} happened: #{ex.message} while resolving address of one of #{names.join(',')}\n"
       end
       addresses
     end

data/lib/domain-probe/version.rb

@@ -1,4 +1,4 @@
 module Probe
-    VERSION = "1.2.0"
+    VERSION = "2.0.0"
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: domain-probe
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 2.0.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-23 00:00:00.000000000 Z
+date: 2012-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70267579051820 !ruby/object:Gem::Requirement
+  requirement: &70212344261720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70267579051820
+  version_requirements: *70212344261720
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70267579051020 !ruby/object:Gem::Requirement
+  requirement: &70212344260960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70267579051020
+  version_requirements: *70212344260960
 - !ruby/object:Gem::Dependency
   name: net-dns
-  requirement: &70267579050440 !ruby/object:Gem::Requirement
+  requirement: &70212344260340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70267579050440
+  version_requirements: *70212344260340
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70267579049620 !ruby/object:Gem::Requirement
+  requirement: &70212344259440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,7 +54,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70267579049620
+  version_requirements: *70212344259440
 description: A simple library to probe the dns records under specific domain, as many
   as possilbe
 email:
@@ -73,6 +73,7 @@ files:
 - lib/domain-probe/guessing_policy.rb
 - lib/domain-probe/policy.rb
 - lib/domain-probe/resolver_patch.rb
+- lib/domain-probe/thread_executor.rb
 - lib/domain-probe/util.rb
 - lib/domain-probe/version.rb
 homepage: http://www.yottaa.com
@@ -89,7 +90,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3749752069138402762
+      hash: -3690899378102268182
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -98,7 +99,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3749752069138402762
+      hash: -3690899378102268182
 requirements: []
 rubyforge_project: domain-probe
 rubygems_version: 1.8.10