domain-probe 1.0.0

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in domain-probe.gemspec
+gemspec

data/README.md

@@ -0,0 +1,3 @@
+= Domain::Probe
+
+Domain::Probe is a simple library to probe the DNS records under specific domain, as many as possible

data/Rakefile

@@ -0,0 +1,16 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'rake'
+require File.expand_path('../lib/domain-probe/version', __FILE__)
+
+task :default
+
+desc 'Builds the gem'
+task :build do
+  sh "gem build domain-probe.gemspec"
+end
+
+desc 'Builds and installs the gem'
+task :install => :build do
+  sh "gem install domain-probe-#{Probe::VERSION}"
+end

data/domain-probe.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "domain-probe/version"
+
+Gem::Specification.new do |s|
+  s.name        = "domain-probe"
+  s.version     = Probe::VERSION
+  s.authors     = ["ijammy"]
+  s.email       = ["mzhang@yottaa.com"]
+  s.homepage    = "http://www.yottaa.com"
+  s.summary     = "A simple library to probe the dns records under specific domain"
+  s.description = "A simple library to probe the dns records under specific domain, as many as possilbe"
+
+  s.rubyforge_project = "domain-probe"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "rake"
+  s.add_runtime_dependency "net-dns"
+  s.add_runtime_dependency "nokogiri"
+end

data/lib/domain-probe/crawling_policy.rb

@@ -0,0 +1,97 @@
+require 'set'
+require 'nokogiri'
+require 'open-uri'
+require 'domain-probe/util'
+require 'domain-probe/policy'
+
+module Probe
+  
+  #consts
+  DOUBLE_SLASH_LENGTH = 2
+  
+  TAG_NAME_A = "a".freeze
+  TAG_NAME_SCRIPT = "script".freeze
+  TAG_NAME_IMG = "img".freeze
+  TAG_NAME_FORM = "form".freeze
+  
+  ATTR_NAME_SRC = "src".freeze
+  ATTR_NAME_ACTION = "action".freeze
+  ATTR_NAME_HREF = "href".freeze
+  
+  #
+  class CrawlingPolicy < Policy
+    
+     #
+     # return a set of possilbe host under specific domain by crawling the URL within HTML tags
+     #
+     def possible_host_under_domain(domain)
+       #sanity check
+       return [].to_set if !domain
+       
+       set = Set.new()
+       begin
+         
+         #choose the default website to try
+         page = Nokogiri::HTML(open(Util.polish_url(domain)))
+       
+         #a lambda to check if the attribute is of an absolute url or not
+         absolute_url_checker = lambda { |element,attribute| 
+           attrib = element[attribute] 
+           attrib && (attrib.downcase =~ /^http[s]?:\/\// || attrib.downcase.start_with?("//") )
+         }
+       
+         #try to find some subdomains by a tags
+         links = page.css(TAG_NAME_A).select{ |element| absolute_url_checker.call(element,ATTR_NAME_HREF) }
+         set |= iterate_elements_with_attr(links,ATTR_NAME_HREF,domain)
+       
+         #try to find some subdomains by script tags
+         scripts = page.css(TAG_NAME_SCRIPT).select{ |element| absolute_url_checker.call(element,ATTR_NAME_SRC) }
+         set |= iterate_elements_with_attr(scripts,ATTR_NAME_SRC,domain)
+       
+         #try to find some subdomains by img tags
+         images = page.css(TAG_NAME_IMG).select{ |element| absolute_url_checker.call(element,ATTR_NAME_SRC) }
+         set |= iterate_elements_with_attr(images,ATTR_NAME_SRC,domain)
+         
+         #try to find some subdomains by img tags
+         forms = page.css(TAG_NAME_FORM).select{ |element| absolute_url_checker.call(element,ATTR_NAME_ACTION) }
+         set |= iterate_elements_with_attr(forms,ATTR_NAME_ACTION,domain)
+         
+       rescue => ex
+         puts "#{ex.class}: #{ex.message}"
+       end
+       set
+     end
+     
+     private
+     
+  
+
+     
+     #
+     #iterate the elements to find a set of host names
+     #
+     def iterate_elements_with_attr(links,attribute,domain)
+        raise ArgumentError, "Both attribute name and domain name are expected" if !attribute || !domain
+        return [].to_set if !links
+        set = Set.new()
+        
+        links.each { |link|
+          next unless link[attribute]
+          head = link[attribute].index("//")
+          next unless head
+          tail = link[attribute].index(domain)
+          next unless tail
+          
+          #slice the host in the url
+          host = link[attribute].slice(head+DOUBLE_SLASH_LENGTH,tail-head-DOUBLE_SLASH_LENGTH)
+          
+          #remove the trailing dot if any
+          host.slice!(host.length-1) if host.end_with? "."
+          set.add host
+        }
+        
+        set
+     end
+  end
+
+end

data/lib/domain-probe/guessing_policy.rb

@@ -0,0 +1,120 @@
+require 'domain-probe/policy'
+module Probe
+  
+  #
+  # Popular host names, in alphabetic order
+  #
+  COMMON_NAMES = %w{
+    ad
+    ads
+    api
+    app
+    application
+    apps
+    ask
+    asset
+    assets
+    au
+    auto
+    
+    baby
+    bbs
+    biz
+    blog
+    build
+    cal
+    calendar
+    cards
+    client
+    club
+    college
+    date
+    dev
+    develop
+    developer
+    dictionary
+    dir
+    direct
+    discovery
+    docs
+    download
+    downloads
+    email
+    english
+    events
+    feed
+    finance
+    food
+    fresh
+    ftp
+    help
+    home
+    hr
+    image
+    images
+    imap
+    
+    info
+    ip
+    iphone
+    local
+    log
+    login
+    love
+    m
+    mail
+    maps
+    media
+    message
+    messages
+    mon
+    money
+    monitor
+    mp3
+    news
+    operation
+    operations
+    page
+    pages
+    php
+    play
+    plus
+    pops
+    production
+    qa
+    read
+    repo
+    resource
+    resources
+    sg
+    shopping
+    sms
+    smtp
+    sports
+    staging
+    static
+    stats
+    status
+    studios
+    support
+    trading
+    video
+    view
+    vip
+    web
+    webmail
+    wiki
+    wireless
+    www
+    www1
+  }
+  
+  class GuessingPolicy < Policy
+    
+     #
+     def possible_host_under_domain(domain)
+       COMMON_NAMES.to_set
+     end
+  end
+  
+end

data/lib/domain-probe/policy.rb

@@ -0,0 +1,17 @@
+module Probe
+  
+  #const declaration
+  WWW_PREFIX = "www."
+  
+  #
+  # The policy interface specification, each child class must implement the possible_host_under_domain method
+  #
+  class Policy
+    
+     def possible_host_under_domain(domain)
+       [].to_set
+     end
+     
+  end
+  
+end

data/lib/domain-probe/resolver_patch.rb

@@ -0,0 +1,34 @@
+require 'net/dns/resolver'
+
+#
+# Hack: the RR in net-dns didn't implement the ==,eql?,hash.methods
+#
+module Net 
+  module DNS
+    class RR
+      class TXT
+        def value
+          @txt.to_s
+        end
+        
+        def get_inspect
+          value
+        end    
+      end
+      
+      def ==(o)
+        if o.is_a? RR
+          self.data == o.data
+        else
+          false
+        end
+      end
+      
+      alias eql? ==
+      
+      def hash
+        self.data.hash
+      end
+    end
+  end
+end

data/lib/domain-probe/util.rb

@@ -0,0 +1,157 @@
+require 'domain-probe/resolver_patch'
+#
+# 
+#
+module Probe
+  
+  #
+  # Utility functions to assist domain-probing
+  #
+  class Util
+    
+    #
+    # Detect the zone name the specific domain belongs to
+    #
+    def self.detect_zone domain
+      
+      return nil if !domain
+      domain = append_trailing_dot_if_necessary domain
+      names = domain.split "."
+      
+      zone = nil
+      begin
+        names.size.times { |index|
+          resovler = Net::DNS::Resolver.new
+          packet = resovler.query(domain, Net::DNS::NS)
+          packet.answer.each { |record|
+              next unless record.name == domain && record.type == "NS"
+              zone = domain
+              break
+          } if packet && packet.answer
+        
+          return remove_trailing_dot_if_any(zone) unless !zone
+          domain = domain.slice(names[index].length + 1,domain.length - names[index].length - 1)
+        } 
+      rescue  => ex
+        puts "#{ex.class}: #{ex.message}"
+      end
+      
+      zone
+    end
+    
+    #
+    # Detect the nameservers against specific zone
+    #
+    def self.detect_nameservers zone
+      return [] if !zone
+      zone = append_trailing_dot_if_necessary zone
+      
+      nameserver_ips = []
+      nameserver_names = []
+      begin
+        resovler = Net::DNS::Resolver.new
+        packet = resovler.query(zone, Net::DNS::NS)
+        packet.answer.each { |record|
+            next unless record.name == append_trailing_dot_if_necessary(zone) && record.type == "NS"
+            nameserver_names << record.nsdname
+            packet.additional.select{ |x| x.name == record.nsdname && x.type == "A" }.each{ |rr|
+                nameserver_ips << rr.address.to_s
+            }
+        } if packet && packet.answer
+        
+        if nameserver_ips.empty? 
+          nameserver_ips += resolve_addresses(nameserver_names)
+        end
+      rescue  => ex
+        puts "#{ex.class}: #{ex.message}"
+      end
+      
+      nameserver_ips
+    end
+    
+    #
+    # Organize the authorities to assist recusive queries
+    #
+    def self.organize_authorities(authorities,additional)
+      
+      return [] if !authorities
+      nameserver_ips = []
+      nameserver_names = []
+      authorities.each{ |authority|
+        next unless authority.type == "NS"
+        nameserver_names << authority.nsdname
+        additional.select{ |x| x.name == authority.nsdname && x.type == "A" }.each{ |rr|
+            nameserver_ips << rr.address.to_s
+        }
+      }
+         
+      if nameserver_ips.empty? 
+        nameserver_ips += resolve_addresses(nameserver_names)
+      end
+      
+      nameserver_ips
+    end
+    
+    #
+    #
+    #
+    def self.polish_url(domain)
+      return nil if !domain
+      zone = detect_zone domain
+     "http://" + (!zone || domain.start_with?(WWW_PREFIX) ? domain : WWW_PREFIX + zone)
+    end
+    
+    
+    #
+    # if the domain is not ended with dot, then append one
+    #
+    def self.append_trailing_dot_if_necessary domain
+       return domain if !domain || domain.end_with?(".")
+       domain + "."
+    end
+    
+    #
+    # removing trailing dot, if there is one
+    #
+    def self.remove_trailing_dot_if_any domain
+       return domain unless !domain || domain.end_with?(".")
+       domain.chop
+    end
+    
+    #
+    # to judge whether the domain is a subdomain of zone.
+    #
+    def self.is_subdomain?(domain,zone)
+      return false if !domain || !zone
+      zone_with_trailing_dot = Util.append_trailing_dot_if_necessary(zone)
+      return false if domain == zone_with_trailing_dot
+      domain.end_with?(zone_with_trailing_dot) && domain[domain.index(zone_with_trailing_dot) - 1,1] == "."
+    end
+    
+    private
+    
+    #
+    # resolve the names to ip addresses
+    #
+    def self.resolve_addresses names
+      
+      return [] unless names && !names.empty?
+      addresses = []
+      begin
+          resovler = Net::DNS::Resolver.new
+          names.each{ |name|
+              packet = resovler.query(name, Net::DNS::A)
+              packet.answer.each { |record|
+                  next unless record.type == "A"
+                  addresses << record.address.to_s
+              }
+          }
+      rescue  => ex
+        puts "#{ex.class}: #{ex.message}"
+      end
+      addresses
+    end
+    
+  end
+  
+end

data/lib/domain-probe/version.rb

@@ -0,0 +1,4 @@
+module Probe
+    VERSION = "1.0.0"
+end
+

data/lib/domain-probe.rb

@@ -0,0 +1,222 @@
+require 'domain-probe/resolver_patch'
+require 'domain-probe/util'
+require 'domain-probe/policy'
+require 'domain-probe/guessing_policy'
+require 'domain-probe/crawling_policy'
+require 'set'
+
+module Probe
+  
+  class DomainProbe
+    
+     MAX_RECURSIVE_QUERY_DEPTH = 6
+     
+     @policies = []
+     
+     class << self
+       attr_accessor :policies
+     end
+     
+     #
+     #
+     def self.register_policy policy
+      raise ArgumentError, "policy need respond_to? possible_host_under_domain" unless policy.kind_of? Policy
+      self.policies << policy
+     end
+     
+     def self.unregister_policy policy
+      self.policies.delete policy
+     end
+     
+     #
+     # collect the records under domain domain
+     #
+     def self.collect_records domain
+       self.new.collect_records domain
+     end
+     
+     #
+     # ----------------------------------------
+     #
+     #
+     #
+     def initialize()
+       @a_records     = []
+       @aaaa_records  = []
+       @cname_records = []
+       @mx_records    = []
+       @mr_records    = []
+       @txt_records   = []
+       @hinfo_records = []
+       @srv_records   = []
+       @ns_records    = []
+       @semaphore     = Mutex.new
+     end
+     
+     attr_accessor :a_records
+     attr_accessor :aaaa_records
+     attr_accessor :cname_records
+     attr_accessor :mx_records
+     attr_accessor :mr_records
+     attr_accessor :txt_records
+     attr_accessor :hinfo_records
+     attr_accessor :srv_records
+     attr_accessor :ns_records
+     attr_accessor :semaphore
+      
+      #
+      # collect the records under domain domain
+      #
+      def collect_records domain
+          raise ArgumentError, "Domain name is expected" unless domain
+          domain = zone = Util.detect_zone(domain)
+          
+          threads = []
+          nameservers = Util.detect_nameservers zone
+          possible_domains = Set.new
+          
+          DomainProbe.policies.each{ |policy|
+            possible_domains |= policy.possible_host_under_domain(zone)
+          }
+       
+          #A/CNAME records
+          possible_domains.each{ |possible_host|
+               possible_domain = possible_host + "." + zone
+
+               threads << threaded_resolve(possible_domain,Net::DNS::A,zone,nameservers) 
+               #AAAA
+               threads << threaded_resolve(possible_domain,Net::DNS::AAAA,zone,nameservers)  
+          }
+          
+          #A/CNAME records
+          threads << threaded_resolve(domain,Net::DNS::A,zone,nameservers) 
+          #AAAA
+          threads << threaded_resolve(domain,Net::DNS::AAAA,zone,nameservers)
+          
+          #MX records
+          threads << threaded_resolve(domain,Net::DNS::MX,zone,nameservers) 
+          
+          #MR records
+          threads << threaded_resolve(domain,Net::DNS::MR,zone,nameservers)
+          
+          #TXT records
+          threads << threaded_resolve(domain,Net::DNS::TXT,zone,nameservers)
+          
+          #HINFO records
+          threads << threaded_resolve(domain,Net::DNS::HINFO,zone,nameservers)
+          
+          #SRV records
+          threads << threaded_resolve(domain,Net::DNS::SRV,zone,nameservers)
+
+          #NS records
+          threads << threaded_resolve(domain,Net::DNS::NS,zone,nameservers)          
+
+          threads.each do |t|
+            t.join
+          end
+
+          self.a_records + self.aaaa_records + self.cname_records + self.mx_records + self.mr_records + self.txt_records + self.hinfo_records + self.srv_records + self.ns_records
+      end
+       
+    private
+
+      #
+      # the method will be called under the threaded execution context
+      # Attention: 
+      # 1. tail recursive call happened when we get an un-authorized reply
+      # 2. depth limit to avoid infinite recursion
+      #
+      def resolve_recursive(domain,type,zone,nameservers,depth)
+        
+        depth += 1
+        return if depth > MAX_RECURSIVE_QUERY_DEPTH
+        single_record_handler = lambda{  |record|
+    
+          if record.name && (record.name == Util.append_trailing_dot_if_necessary(zone) || Util.is_subdomain?(record.name,zone))
+           #A little overhead here
+           self.semaphore.synchronize {
+             case record.type
+             when "A"      
+               self.a_records << record unless self.a_records.include? record
+             when "AAAA"
+               self.aaaa_records << record unless self.aaaa_records.include? record
+             when "CNAME"
+               self.cname_records << record unless self.cname_records.include? record               
+             when "NS"
+               self.ns_records << record unless self.ns_records.include? record
+             when "MX"
+               self.mx_records << record unless self.mx_records.include? record
+             when "MR"
+               self.mr_records << record unless self.mr_records.include? record 
+             when "TXT"
+               self.txt_records << record unless self.txt_records.include? record
+             when "SRV"
+               self.srv_records << record unless self.srv_records.include? record
+             when "HINFO"
+               self.hinfo_records << record unless self.hinfo_records.include? record   
+             else
+               #nothing to do?
+             end
+           } 
+         end
+       }
+     
+       #
+       begin
+         
+         resolver = nameservers.empty? ? Net::DNS::Resolver.new : Net::DNS::Resolver.new(:nameserver => nameservers)
+         resolver.log_level = Net::DNS::ERROR
+         packet = resolver.query(domain, type)
+         return unless packet
+         
+         if packet.header.auth?
+            packet.answer.each &single_record_handler if packet.answer
+            packet.authority.each &single_record_handler if packet.authority
+            packet.additional.each &single_record_handler if packet.additional
+         elsif packet.answer.empty? && !packet.authority.empty?
+            #following down
+            nameservers = Util.organize_authorities(packet.authority,packet.additional)
+            #tail recursive
+            resolve_recursive(domain,type,zone,nameservers,depth)
+         end
+         
+       rescue =>ex
+         puts "#{ex.class}: #{ex.message}"
+       end
+      end
+
+      #
+      # The thread should benifit us, IF NOT, disable it
+      #
+      def threaded_resolve(domain,type,zone,nameservers)
+          Thread.new(domain,type,zone,nameservers) { |domain,type,zone,nameservers|
+              depth = 0
+              resolve_recursive(domain,type,zone,nameservers,depth)
+          }
+      end
+      
+      register_policy GuessingPolicy.new
+      register_policy CrawlingPolicy.new
+  end
+  
+   #
+   #
+   #
+   def collect_records domain 
+     DomainProbe.collect_records domain
+   end
+   
+   def collect_records_print_beautifully domain
+     start = Time.now.to_i
+     records = DomainProbe.collect_records(domain)
+     finish = Time.now.to_i
+     print "-----------------------","\n"
+     records.each{ |record|
+        print record, "\n"
+     }
+     print "-----------------------","\n"
+     print "Time elapsed:\t",finish - start,"seconds. \n"
+     nil
+   end
+   
+end

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification
+name: domain-probe
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- ijammy
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-04-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70294597444060 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70294597444060
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70294597442500 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70294597442500
+- !ruby/object:Gem::Dependency
+  name: net-dns
+  requirement: &70294597440720 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70294597440720
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &70294597439640 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70294597439640
+description: A simple library to probe the dns records under specific domain, as many
+  as possilbe
+email:
+- mzhang@yottaa.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- domain-probe.gemspec
+- lib/domain-probe.rb
+- lib/domain-probe/crawling_policy.rb
+- lib/domain-probe/guessing_policy.rb
+- lib/domain-probe/policy.rb
+- lib/domain-probe/resolver_patch.rb
+- lib/domain-probe/util.rb
+- lib/domain-probe/version.rb
+homepage: http://www.yottaa.com
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: domain-probe
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: A simple library to probe the dns records under specific domain
+test_files: []