dnstraverse 0.0.1 → 0.1.0

data/lib/dnstraverse/message_utility.rb

@@ -24,15 +24,15 @@ module DNSTraverse
       warnings = Array.new
       if args[:want_recursion] then
         if not msg.header.ra then
-          warnings.push "#{msg.answerfrom} doesn't allow recursion"
+          warnings << "#{msg.answerfrom} doesn't allow recursion"
         end
       else
         if msg.header.ra then
-          warnings.push "#{msg.answerfrom} allows recursion"
+          warnings << "#{msg.answerfrom} allows recursion"
         end
       end
       if msg.header.tc then
-        warnings.push "#{msg.answerfrom} sent truncated packet"
+        warnings << "#{msg.answerfrom} sent truncated packet"
       end
       for warn in warnings do
         Log.warn { warn }
@@ -134,11 +134,11 @@ module DNSTraverse
         type = rr.type.to_s
         klass = rr.klass.to_s
         if type.casecmp('NS') == 0 && klass.casecmp('IN') == 0
-          ns.push rr
+          ns << rr
         elsif type.casecmp('SOA') == 0 && klass.casecmp('IN') == 0
-          soa.push rr
+          soa << rr
         else
-          other.push rr
+          other << rr
         end
       end
       return ns, soa, other      
@@ -173,26 +173,31 @@ module DNSTraverse
       return false
     end
     
-    def msg_cacheable(msg, bailiwick, type = :both)
-      good, bad = Array.new, Array.new
+    def msg_cacheable(msg, bailiwick, type = :all)
+      good, bad, other = [], [], []
       bw = bailiwick.to_s
       bwend = "." + bw
-      for section in [:additional, :authority] do
+      for section in [:answer, :authority, :additional] do
         for rr in msg.send(section) do
-          name = rr.name.to_s
-          if bailiwick.nil? or name.casecmp(bw) == 0 or
-            name =~ /#{bwend}$/i then
-            good.push rr
+          if rr.type.to_s == "OPT" then
+            other << rr
           else
-            bad.push rr
+            name = rr.name.to_s
+            if bailiwick.nil? or name.casecmp(bw) == 0 or
+              name =~ /#{bwend}$/i then
+                good << rr
+            else
+              bad << rr
+            end
           end
         end
       end
       good.map {|x| Log.debug { "Records within bailiwick: " + x.to_s } }
       bad.map {|x| Log.debug { "Records outside bailiwick: " + x.to_s } }
+      other.map {|x| Log.debug { "Other records discarded: " + x.to_s } }
       return good if type == :good
       return bad if type == :bad
-      return good, bad
+      return good, bad, other
     end
     
   end

data/lib/dnstraverse/referral.rb

@@ -17,6 +17,7 @@ require 'dnstraverse/response'
 require 'dnstraverse/response_noglue'
 require 'dnstraverse/info_cache'
 require 'dnstraverse/decoded_query_cache'
+require 'dnstraverse/summary_stats'
 
 module DNSTraverse
   
@@ -24,9 +25,11 @@ module DNSTraverse
     include MessageUtility
     
     attr_reader :server, :serverips, :qname, :qclass, :qtype, :nsatype
-    attr_reader :refid, :message, :infocache, :parent, :bailiwick, :stats
+    attr_reader :refid, :refkey, :infocache, :parent, :bailiwick
     attr_reader :warnings, :children, :parent_ip
     attr_reader :decoded_query_cache
+    attr_reader :responses
+    attr_reader :stats
     
     EMPTY_ARRAY = [].freeze
     
@@ -88,7 +91,8 @@ module DNSTraverse
       @infocache = args[:infocache] || DNSTraverse::InfoCache.new
       @roots = args[:roots]
       @resolves = nil # Array of referral objects for resolving phase
-      @refid = args[:refid] || ''
+      @refid = args[:refid] || '' # node identifier, e.g. 1.2.1
+      @refkey = args[:refkey] || '' # node key, e.g. 3.4.3 (maximums)
       @server = args[:server] || nil # nil for the root-root server
       @serverips = args[:serverips] || nil
       @responses = Hash.new # responses/exception for each IP in @serverips
@@ -105,6 +109,7 @@ module DNSTraverse
       @serverweights = Hash.new # key is IP
       @warnings = Array.new # warnings will be placed here
       @processed = false # flag for processed? method
+      @calculated = false # flag for calculated? method
       raise "Must pass Resolver" unless @resolver
       @infocache.add_hints('', args[:roots]) if args[:roots] # add root hints
       unless @decoded_query_cache then
@@ -168,7 +173,6 @@ module DNSTraverse
                     " of #{bailiwick} - no glue record provided" }
         return Array.new
       end
-      refid = "#{@refid}.0"
       child_refid = 1
       starters, newbailiwick = @infocache.get_startservers(@server)
       Log.debug { "Resolving #{@server} type #{@nsatype} " }
@@ -177,8 +181,10 @@ module DNSTraverse
                           :serverips => starter[:ips],
                           :referral_resolution => true,
                           :qname => @server, :qclass => 'IN',
-        :qtype => @nsatype, :bailiwick => newbailiwick,
-        :refid => "#{refid}.#{child_refid}")
+                          :qtype => @nsatype,
+			  :bailiwick => newbailiwick,
+                          :refid => "#{@refid}.0.#{child_refid}",
+			  :refkey => "#{@refkey}.0.#{starters.count}")
          (@resolves||= []) << r
         child_refid+= 1
       end
@@ -249,6 +255,7 @@ module DNSTraverse
         @stats.each_pair do |key, data|
           Log.debug { sprintf "Answer: %.2f%% %s\n", data[:prob] * 100, key }
         end
+        @calculated = true
         return
       end
       for ip in @serverips do
@@ -277,12 +284,17 @@ module DNSTraverse
       @stats.each_pair do |key, data|
         Log.debug { sprintf "Answer: %.2f%% %s\n", data[:prob] * 100, key }
       end
+      @calculated = true
     end
     
     def processed?
       return @processed
     end
     
+    def calculated?
+      return @calculated
+    end
+    
     def resolved?
       # root-root is always resolved, otherwise check we have IP addresses
       return true if is_rootroot?
@@ -314,15 +326,16 @@ module DNSTraverse
     
     def process_add_roots(args)
       Log.debug { "Special case processing, addding roots as referrals" }
-      refid_prefix = @refid == '' ? '' : "#{@refid}."
-      refid = 1
+      dot = @refid == '' ? '' : "."
+      child_refid = 1
       starters = (@infocache.get_startservers('', @nsatype))[0]
       @children[:rootroot] = Array.new # use 'rootroot' instead of IP address
       for root in starters do
         r = make_referral(:server => root[:name], :serverips => root[:ips],
-                          :refid => "#{refid_prefix}#{refid}")
+                          :refid => "#{@refid}#{dot}#{child_refid}",
+			  :refkey => "#{@refkey}#{dot}#{starters.count}")
         @children[:rootroot].push r
-        refid+= 1
+        child_refid+= 1
       end
     end
     
@@ -390,7 +403,8 @@ module DNSTraverse
         refargs = {
           :server => starter[:name],
           :serverips => starter[:ips],
-          :refid => "#{refid}.#{child_refid}"
+          :refid => "#{@refid}.#{child_refid}",
+	  :refkey => "#{@refkey}.#{starters.count}"
         }.merge(args)
         children.push make_referral(refargs)
         child_refid+= 1
@@ -400,6 +414,7 @@ module DNSTraverse
     
     def make_referral(args)
       raise "Must pass new refid" unless args[:refid]
+      raise "Must pass new refkey" unless args[:refkey]
       refargs = { :qname => @qname, :qclass => @qclass,
         :qtype => @qtype, :nsatype => @nsatype, :infocache => @infocache,
         :referral_resolution => @referral_resolution,
@@ -437,7 +452,7 @@ module DNSTraverse
           when :error
           puts "#{response.error_message} at #{where}"
           when :nodata
-          puts "NODATA (for this type) at #{where})"
+          puts "NODATA (for this type) at #{where}"
           when :answered
           puts "Answer from #{where}"
           if results then
@@ -457,7 +472,12 @@ module DNSTraverse
         end
       end
     end
+
+    # Returns a SummaryStats object
+    def summary_stats
+      return nil unless calculated?
+      @summary_stats_object||= DNSTraverse::SummaryStats.new(self)
+    end
     
   end
-  
 end

data/lib/dnstraverse/response.rb

@@ -65,6 +65,8 @@ module DNSTraverse
     def evaluate
       @status = @decoded_query.status # use this as a base
       if @status != :exception
+        # XXX order of cacheable_good is answer, authority, additional
+        # perhaps we should add some checking for overlap between sections?
         @infocache.add(@decoded_query.cacheable_good)
       end
       case @decoded_query.status
@@ -89,4 +91,4 @@ module DNSTraverse
       end
     end
   end
-end
+end

data/lib/dnstraverse/summary_stats.rb

@@ -0,0 +1,94 @@
+module DNSTraverse
+  class SummaryStats
+
+    def initialize(referral)
+      @summary_stats = get_summary_stats(referral)
+      @answer_stats = get_answer_stats(referral)
+    end
+
+    # summary returns a hash summarising the results
+    #
+    # the key is the type of result, e.g.:
+    #   :exception, :noglue, :nodata, :answered, etc.
+    # the value is a hash containing:
+    #   :prob #=> probability of getting this result
+    # all probabilities will add up to 1
+    #
+    # for example
+    # {
+    #   :exception => { :prob => 0.5 },
+    #   :answered => { :prob => 0.5 }
+    # }
+    def summary
+      @summary_stats
+    end
+
+    # each_summary takes a block and yields the type and summary info
+    #
+    # for example
+    # stats.each_summary do |type, sinfo|
+    #   puts "#{prob} = #{sinfo[:prob]}"
+    # end
+    def each_summary
+      @summary_stats.each_pair do |type, sinfo|
+        yield type, sinfo
+      end
+    end
+
+    # answers returns a hash summarising the answers referred to
+    # by the key :answered in the summary statistics
+    # 
+    # the key is the rdata string (e.g. 192.168.0.1)
+    # the value is a hash containing:
+    #   :prob #=> probability of getting this result
+    #   :rrs #=> array of Dnsruby::RR records
+    # all probabilities add up to the chance of getting an answer
+    #   (e.g. get_summary_stats()[:answered][:prob] )
+    #
+    # for example
+    # {
+    #   '192.168.0.1' => { :prob => 0.2, :rrs => [...] },
+    #   '192.168.0.2' => { :prob => 0.3, :rrs => [...] }
+    # }
+    def answers
+      @answer_stats
+    end
+
+    # each_answer takes a block and yields the probability and RR array
+    #
+    # for example
+    # stats.each_answer do |prob, records|
+    #   puts "#{prob} = #{records.join(',')}"
+    # end
+    def each_answer
+      @answer_stats.each_pair do |type, ainfo|
+        yield ainfo[:prob], ainfo[:rr]
+      end
+    end
+
+    private
+    def get_summary_stats(referral)
+      s = {}
+      referral.stats.each_pair do |key, data|
+        response = data[:response]
+        type = response.status
+        s[type]||= { :prob => 0.0 }
+        s[type][:prob]+= data[:prob]
+      end
+      return s
+    end
+
+    def get_answer_stats(referral)
+      a = {}
+      referral.stats.each_pair do |key, data|
+        response = data[:response]
+        if response.status == :answered then
+          key = response.answers.map {|rr| rr.rdata_to_string }.sort.join("@@@")
+          a[key]||= { :prob => 0.0, :rr => response.answers }
+          a[key][:prob]+= data[:prob]
+        end
+      end
+      return a
+    end
+  end
+end

data/lib/dnstraverse/traverser.rb

@@ -13,7 +13,7 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-gem 'dnsruby', '>=1.26'
+gem 'dnsruby', '>=1.30'
 require 'dnsruby'
 require 'dnstraverse/info_cache'
 require 'dnstraverse/log'
@@ -61,7 +61,6 @@ module DNSTraverse
         :ignore_truncation => ignore_truncation, :src_address => srcaddr,
         :udp_size => udpsize.to_i, :packet_timeout => packet_timeout }
       Log.debug { "Creating remote resolver object"}
-      CachingResolver.use_eventmachine(false)
       @resolver = CachingResolver.new(resargs) # used for set nameservers
       @resolver.udp_size = udpsize.to_i
       Log.debug { "Creating local resolver object"}
@@ -71,6 +70,13 @@ module DNSTraverse
       self
     end
     
+    def report_progress(refobj, opts = {})
+      stage = opts[:stage] or raise "must pass option :stage"
+      refres = refobj.referral_resolution?
+      p = (refres == true ? @progress_resolve : @progress_main)
+      p.call(:state => @state, :referral => refobj, :stage => stage)
+    end
+
     ### change to get_all or something?
     def get_a_root(args)
       aaaa = args[:aaaa] || false
@@ -139,25 +145,19 @@ module DNSTraverse
         r = stack.pop
         Log.debug { "running on stack entry #{r}" }
         case r
-          when :calc_resolve
+        when :calc_resolve
           r = stack.pop
           r.resolve_calculate
-          refres = r.referral_resolution?
-          p = (refres == true ? @progress_resolve : @progress_main)
-          p.call(:state => @state, :referral => r, :stage => :resolve)
+          report_progress r, :stage => :resolve
           stack << r # now need to process
           next
-          when :calc_answer
+        when :calc_answer
           r = stack.pop
           r.answer_calculate
-          refres = r.referral_resolution?
-          p = (refres == true ? @progress_resolve : @progress_main)
-          p.call(:state => @state, :referral => r, :stage => :answer)
+          report_progress r, :stage => :answer
           r.cleanup(cleanup)
           if @fast then
             # store away in @answered hash so we can lookup later
-            # XXX fast method should use IP and not server name?
-            # or maybe we should append IPs to end... 
             key = "#{r.qname}:#{r.qclass}:#{r.qtype}:#{r.server}:#{r.txt_ips_verbose}"
             key.downcase!
             Log.debug { "Fast mode cache store: #{key}" }
@@ -170,14 +170,14 @@ module DNSTraverse
           end
           next
         else
-          refres = r.referral_resolution?
-          p = (refres == true ? @progress_resolve : @progress_main)
-          p.call(:state => @state, :referral => r, :stage => :start)
+          report_progress r, :stage => :start
         end
         unless r.resolved? then
           # get resolve Referral objects, place on stack with placeholder
           stack << r << :calc_resolve
-          stack.push(*r.resolve({}).reverse)
+          referrals = r.resolve({})
+          referrals.each { |c| report_progress c, :stage => :new }
+          stack.push(*referrals.reverse) # XXX
           next
         end
         unless r.processed? then
@@ -199,15 +199,14 @@ module DNSTraverse
               if @answered.key?(key) and (not c.noglue?) then
                 Log.debug { "Fast method - completed #{c}" }
                 r.replace_child(c, @answered[key])
-                refres = r.referral_resolution?
-                p = (refres == true ? @progress_resolve : @progress_main)
-                p.call(:state => @state, :referral => c, :stage => :answer_fast)
+                report_progress c, :stage => :answer_fast
               else
                 newchildren << c
               end
             end
             children = newchildren
           end
+          children.each { |c| report_progress c, :stage => :new }
           stack.push(*children.reverse)
           next
         end
@@ -260,7 +259,7 @@ module DNSTraverse
           Log.warn { "Failed to resolve #{rr.domainname} type #{qtype}" }
           next
         end
-        roots.push({ :name => rr.domainname, :ips => ips })
+        roots.push({ :name => rr.domainname.to_s, :ips => ips })
       end
       Log.debug { "find_roots exit, #{roots.map { |x| x[:name] }.join(', ') }" }
       return roots
@@ -275,6 +274,7 @@ module DNSTraverse
       r = Referral.new(:qname => qname, :qtype => qtype, :roots => args[:roots],
                        :maxdepth => maxdepth, :resolver => @resolver,
                        :nsatype => 'A')
+      report_progress r, :stage => :new
       run(r, :cleanup => cleanup)
       Log.debug { "run_query exit" }
       return r

data/lib/dnstraverse/version.rb

@@ -0,0 +1,8 @@
+module DNSTraverse
+  module Version
+    MAJOR = 0
+    MINOR = 1
+    PATCH = 0
+    STRING = "#{MAJOR}.#{MINOR}.#{PATCH}"
+  end
+end