domain-probe 1.0.0 → 1.2.0

data/README.md

@@ -1,3 +1,34 @@
 = Domain::Probe
 
-Domain::Probe is a simple library to probe the DNS records under specific domain, as many as possible
+Domain::Probe is a simple library to probe the DNS records under specific domain, as many as possible
+
+== Features
+
+* Queries from authority name servers
+* Recursive resolving
+* Zone name detection
+
+
+== Requirements
+
+* Ruby >= 1.8.7
+
+
+== Install
+
+Just use RubyGems:
+  
+  $ gem install domain-probe
+
+If you want to install from source, you can use Rake:
+
+  $ rake install
+
+
+== How to use
+
+1. require 'domain-probe'
+2. include Probe
+3. collect_records "www.facebook.com"
+
+    

data/lib/domain-probe.rb

@@ -32,7 +32,14 @@ module Probe
      # collect the records under domain domain
      #
      def self.collect_records domain
-       self.new.collect_records domain
+       self.new(domain).collect_records
+     end
+     
+     #
+     # collect the ns records
+     #
+     def self.collect_ns_records domain
+       self.new(domain).collect_ns_records
      end
      
      #
@@ -40,7 +47,8 @@ module Probe
      #
      #
      #
-     def initialize()
+     def initialize(domain)
+       @domain = domain.downcase
        @a_records     = []
        @aaaa_records  = []
        @cname_records = []
@@ -49,6 +57,7 @@ module Probe
        @txt_records   = []
        @hinfo_records = []
        @srv_records   = []
+       @spf_records    = []
        @ns_records    = []
        @semaphore     = Mutex.new
      end
@@ -62,14 +71,17 @@ module Probe
      attr_accessor :hinfo_records
      attr_accessor :srv_records
      attr_accessor :ns_records
+     attr_accessor :spf_records
      attr_accessor :semaphore
+     attr_accessor :domain
       
       #
       # collect the records under domain domain
       #
-      def collect_records domain
-          raise ArgumentError, "Domain name is expected" unless domain
-          domain = zone = Util.detect_zone(domain)
+      def collect_records
+          raise ArgumentError, "Domain name is expected" unless self.domain
+          domain = zone = Util.detect_zone(self.domain)
+          return [] if !zone
           
           threads = []
           nameservers = Util.detect_nameservers zone
@@ -108,6 +120,9 @@ module Probe
           #SRV records
           threads << threaded_resolve(domain,Net::DNS::SRV,zone,nameservers)
 
+          #SPF records
+          threads << threaded_resolve(domain,Net::DNS::SPF,zone,nameservers)
+          
           #NS records
           threads << threaded_resolve(domain,Net::DNS::NS,zone,nameservers)          
 
@@ -115,7 +130,36 @@ module Probe
             t.join
           end
 
-          self.a_records + self.aaaa_records + self.cname_records + self.mx_records + self.mr_records + self.txt_records + self.hinfo_records + self.srv_records + self.ns_records
+          results = {
+            "a" => self.a_records,
+            "aaaa" => self.aaaa_records,
+            "cname" => self.cname_records,
+            "mx" => self.mx_records,
+            "mr" => self.mr_records,
+            "txt" => self.txt_records,
+            "hinfo" => self.hinfo_records,
+            "srv" => self.srv_records,
+            "ns" => self.ns_records,
+            "spf" => self.spf_records
+          }          
+      end
+      
+      def collect_ns_records
+         raise ArgumentError, "Domain name is expected" unless self.domain
+         domain = zone = Util.detect_zone(self.domain)
+         return [] if !zone
+          
+         threads = []
+         nameservers = Util.detect_nameservers zone
+           
+         #NS records
+         threads << threaded_resolve(domain,Net::DNS::NS,zone,nameservers)          
+
+         threads.each do |t|
+           t.join
+         end
+            
+        self.ns_records
       end
        
     private
@@ -137,23 +181,25 @@ module Probe
            self.semaphore.synchronize {
              case record.type
              when "A"      
-               self.a_records << record unless self.a_records.include? record
+               self.a_records << record.to_hash(zone) unless self.a_records.include? record.to_hash(zone)
              when "AAAA"
-               self.aaaa_records << record unless self.aaaa_records.include? record
+               self.aaaa_records << record.to_hash(zone) unless self.aaaa_records.include? record.to_hash(zone)
              when "CNAME"
-               self.cname_records << record unless self.cname_records.include? record               
+               self.cname_records << record.to_hash(zone) unless self.cname_records.include? record.to_hash(zone)               
              when "NS"
-               self.ns_records << record unless self.ns_records.include? record
+               self.ns_records << record.to_hash(zone) unless self.ns_records.include? record.to_hash(zone)
              when "MX"
-               self.mx_records << record unless self.mx_records.include? record
+               self.mx_records << record.to_hash(zone) unless self.mx_records.include? record.to_hash(zone)
              when "MR"
-               self.mr_records << record unless self.mr_records.include? record 
+               self.mr_records << record.to_hash(zone) unless self.mr_records.include? record.to_hash(zone) 
              when "TXT"
-               self.txt_records << record unless self.txt_records.include? record
+               self.txt_records << record.to_hash(zone) unless self.txt_records.include? record.to_hash(zone)
              when "SRV"
-               self.srv_records << record unless self.srv_records.include? record
+               self.srv_records << record.to_hash(zone) unless self.srv_records.include? record.to_hash(zone)
              when "HINFO"
-               self.hinfo_records << record unless self.hinfo_records.include? record   
+               self.hinfo_records << record.to_hash(zone) unless self.hinfo_records.include? record.to_hash(zone) 
+             when "SPF"
+                self.spf_records << record.to_hash(zone) unless self.spf_records.include? record.to_hash(zone)  
              else
                #nothing to do?
              end
@@ -169,11 +215,11 @@ module Probe
          packet = resolver.query(domain, type)
          return unless packet
          
-         if packet.header.auth?
+         if packet.header.auth? || !packet.answer.empty?
             packet.answer.each &single_record_handler if packet.answer
             packet.authority.each &single_record_handler if packet.authority
             packet.additional.each &single_record_handler if packet.additional
-         elsif packet.answer.empty? && !packet.authority.empty?
+         elsif !packet.authority.empty?
             #following down
             nameservers = Util.organize_authorities(packet.authority,packet.additional)
             #tail recursive
@@ -206,17 +252,8 @@ module Probe
      DomainProbe.collect_records domain
    end
    
-   def collect_records_print_beautifully domain
-     start = Time.now.to_i
-     records = DomainProbe.collect_records(domain)
-     finish = Time.now.to_i
-     print "-----------------------","\n"
-     records.each{ |record|
-        print record, "\n"
-     }
-     print "-----------------------","\n"
-     print "Time elapsed:\t",finish - start,"seconds. \n"
-     nil
+   def collect_ns_records domain 
+     DomainProbe.collect_ns_records domain
    end
    
 end

data/lib/domain-probe/crawling_policy.rb

@@ -64,9 +64,6 @@ module Probe
      
      private
      
-  
-
-     
      #
      #iterate the elements to find a set of host names
      #
@@ -77,17 +74,19 @@ module Probe
         
         links.each { |link|
           next unless link[attribute]
-          head = link[attribute].index("//")
+          attribute_value = link[attribute].downcase
+          head = attribute_value.index("//")
           next unless head
-          tail = link[attribute].index(domain)
-          next unless tail
+          tail = attribute_value.index(domain)
+          next unless tail && (tail > head)
           
           #slice the host in the url
           host = link[attribute].slice(head+DOUBLE_SLASH_LENGTH,tail-head-DOUBLE_SLASH_LENGTH)
           
-          #remove the trailing dot if any
-          host.slice!(host.length-1) if host.end_with? "."
-          set.add host
+          next unless host.end_with? "."
+          
+          host.chop!#remove the trailing dot if any
+          set.add host unless host.empty?
         }
         
         set

data/lib/domain-probe/resolver_patch.rb

@@ -1,12 +1,93 @@
 require 'net/dns/resolver'
 
 #
-# Hack: the RR in net-dns didn't implement the ==,eql?,hash.methods
+# Hack: 
+# 1.the RR in net-dns didn't implement the ==,eql?,hash.methods
+# 2.Monkey-patch to support SPF records
 #
 module Net 
   module DNS
     class RR
+      
+      #
+      # removing trailing dot, if there is one
+      #
+      def remove_trailing_dot_if_any domain
+         return domain.chop if domain && domain.end_with?(".")
+         domain
+      end 
+      
+      def to_hash zone
+        cooked_value = name
+        if zone
+          zone << "." unless zone.end_with? "."
+          if name == zone
+            cooked_value = "@"
+          else
+            index = name.index(zone)
+            cooked_value = name[0,index - 1] if index && index > 0
+          end
+        end
+        
+        {:name => cooked_value, :ttl => ttl}
+      end
+      
+  
+      
+      class A
+        def to_hash zone
+          super(zone).merge({:address => address.to_s})
+        end
+      end
+      
+      class CNAME
+        def to_hash zone
+          super(zone).merge({:cname => remove_trailing_dot_if_any(cname)})
+        end
+      end
+      
+      class MX
+        def to_hash zone
+          super(zone).merge({:exchange => remove_trailing_dot_if_any(exchange),:preference => preference.to_s})
+        end
+      end
+      
+      class MR
+        def to_hash zone
+          super(zone).merge({:newname => remove_trailing_dot_if_any(newname)})
+        end
+      end
+      
+      class HINFO
+        def to_hash zone
+          super(zone).merge({:cpu => cpu,:os => os})
+        end
+      end
+      
+      class SRV
+        def to_hash zone
+          super(zone)
+        end
+      end
+      
+      class NS
+        def to_hash zone
+          super(zone).merge({:nsdname => remove_trailing_dot_if_any(nsdname)})
+        end
+      end
+      
+      class AAAA
+        def to_hash zone
+          super(zone).merge({:address => address.to_s})
+        end
+      end
+      
       class TXT
+        
+        def to_hash zone
+          super(zone).merge({:value => txt})
+        end
+        
         def value
           @txt.to_s
         end
@@ -29,6 +110,24 @@ module Net
       def hash
         self.data.hash
       end
+      
+      class SPF < Net::DNS::RR::TXT
+
+         private
+            def set_type
+              @type = Net::DNS::RR::Types.new("SPF")
+            end
+      end
+      
+      class Types
+         TYPES['SPF'] = 99
+      end
+      
+    end
+    
+    module QueryTypes
+     SPF    = 99
     end
+    
   end
 end

data/lib/domain-probe/util.rb

@@ -22,9 +22,9 @@ module Probe
       begin
         names.size.times { |index|
           resovler = Net::DNS::Resolver.new
-          packet = resovler.query(domain, Net::DNS::NS)
+          packet = resovler.query(domain, Net::DNS::SOA)
           packet.answer.each { |record|
-              next unless record.name == domain && record.type == "NS"
+              next unless record.name == domain && record.type == "SOA"
               zone = domain
               break
           } if packet && packet.answer
@@ -114,8 +114,8 @@ module Probe
     # removing trailing dot, if there is one
     #
     def self.remove_trailing_dot_if_any domain
-       return domain unless !domain || domain.end_with?(".")
-       domain.chop
+       return domain.chop if domain && domain.end_with?(".")
+       domain
     end
     
     #

data/lib/domain-probe/version.rb

@@ -1,4 +1,4 @@
 module Probe
-    VERSION = "1.0.0"
+    VERSION = "1.2.0"
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: domain-probe
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-21 00:00:00.000000000 Z
+date: 2012-04-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70294597444060 !ruby/object:Gem::Requirement
+  requirement: &70267579051820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70294597444060
+  version_requirements: *70267579051820
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70294597442500 !ruby/object:Gem::Requirement
+  requirement: &70267579051020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70294597442500
+  version_requirements: *70267579051020
 - !ruby/object:Gem::Dependency
   name: net-dns
-  requirement: &70294597440720 !ruby/object:Gem::Requirement
+  requirement: &70267579050440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70294597440720
+  version_requirements: *70267579050440
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70294597439640 !ruby/object:Gem::Requirement
+  requirement: &70267579049620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,7 +54,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70294597439640
+  version_requirements: *70267579049620
 description: A simple library to probe the dns records under specific domain, as many
   as possilbe
 email:
@@ -87,12 +87,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 3749752069138402762
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 3749752069138402762
 requirements: []
 rubyforge_project: domain-probe
 rubygems_version: 1.8.10