dnstraverse 0.0.1

data/lib/dnstraverse/info_cache.rb

@@ -0,0 +1,108 @@
+#    DNSTraverse traverses the DNS to show statistics and information
+#    Copyright (C) 2008 James Ponder
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, version 3 of the License.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'dnstraverse/log'
+
+module DNSTraverse
+  class InfoCache
+    
+    attr_reader :parent
+    
+    private
+    
+    def key(rr)
+      return "#{rr.name}:#{rr.klass}:#{rr.type}".downcase
+    end
+    
+    public
+
+    def initialize(parent = nil)
+      @parent = parent
+      @data = Hash.new
+      self
+    end
+    
+    # adds the resource records, clearing out any existing entries with the
+    # same details
+    def add(rrs)
+      rrs.each {|rr| @data[key(rr)] = Array.new } # clear out
+      for rr in rrs do
+        @data[key(rr)].push rr
+        Log.debug { "Adding to infocache: #{rr}" }
+      end
+      return nil
+    end
+    
+    # array of hashes containing :name (server name) and :ips (array of strings)
+    # set domain to '' for setting root hints
+    def add_hints(domain, ns)
+      rrs = Array.new
+      for server in ns do
+        rrs.push Dnsruby::RR.create(:name => domain, :ttl => 0,
+                                    :type => 'NS', :domainname => server[:name])
+        for ip in server[:ips] do
+          type = (ip.to_s =~ /\A(\d+)\.(\d+)\.(\d+)\.(\d+)\z/) ? 'A' : 'AAAA'
+          rrs.push Dnsruby::RR.create(:type => type, :ttl => 0,
+                                      :name => server[:name], :address => ip)
+        end
+      end
+      return add(rrs)
+    end
+    
+    def get?(args)
+      qclass = args[:qclass] || 'IN'
+      gkey = "#{args[:qname]}:#{qclass}:#{args[:qtype]}".downcase
+      if @data.has_key?(gkey) then
+        Log.debug { "Infocache recall: " + @data[gkey].join(', ')}
+        return @data[gkey] # returns Array
+      end
+      return nil unless parent
+      return parent.get?(args)
+    end
+    
+    def get_ns?(domain) # get an appropriate ns based on domain
+      domain = domain.to_s
+      while true do
+        Log.debug { "Infocache get_ns? checking NS records for '#{domain}'" }
+        rrs = get?(:qname => domain, :qtype => 'NS')
+        return rrs if rrs
+        if domain == '' then
+          raise "No nameservers available for #{domain} -- no root hints set??"
+        end
+        domain = (i = domain.index('.')) ? domain[i+1..-1] : ''
+      end
+    end
+    
+    def get_startservers(domain, nsatype = 'A')
+      Log.debug { "Getting startservers for #{domain}/#{nsatype}" }
+      newbailiwick = nil
+      # search for best NS records in authority cache based on this domain name
+      ns = get_ns?(domain)
+      starters = Array.new
+      # look up in additional cache corresponding IP addresses if we know them
+      for rr in ns do
+        nameserver = rr.domainname.to_s
+        iprrs = get?(:qname => nameserver, :qtype => nsatype)
+        ips = iprrs ? iprrs.map {|iprr| iprr.address.to_s } : nil
+        starters.push({ :name => nameserver, :ips => ips })
+      end
+      newbailiwick = ns[0].name.to_s
+      Log.debug { "For domain #{domain} using start servers: " +
+        starters.map { |x| x[:name] }.join(', ') }
+      return starters, newbailiwick
+    end
+
+  end
+end

data/lib/dnstraverse/log.rb

@@ -0,0 +1,55 @@
+#    DNSTraverse traverses the DNS to show statistics and information
+#    Copyright (C) 2008 James Ponder
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, version 3 of the License.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'logger'
+
+module Log
+  include Logger::Severity
+  
+  class Formatter
+    Format = "%s, [%s #%d] %5s -- %s: %s\n"
+    
+    def call(severity, time, progname, msg)
+      t = time.strftime("%Y-%m-%d %H:%M:%S.") << "%06d" % time.usec
+      #t = ""
+      msg2str(msg).split(/\n/).map do |m|
+        Format % [severity[0..0], t, $$, severity, progname, m]
+      end
+    end
+    
+    def msg2str(msg)
+      case msg
+        when ::Exception
+        "#{ msg.message } (#{ msg.class })\n" <<
+         (msg.backtrace || []).join("\n")
+      else
+        msg.to_s
+      end
+    end
+  end
+  
+  def self.level=(l)
+    @@logger.level = l
+  end
+  def self.logger=(logger)
+    @@logger = logger
+  end
+  def self.method_missing(key, *args, &b)
+    @@logger.send(key, *args, &b)
+  end
+  @@logger = Logger.new(STDERR)
+  @@logger.formatter = Formatter.new
+  @@logger.level = Logger::FATAL
+end

data/lib/dnstraverse/message_utility.rb

@@ -0,0 +1,199 @@
+#    DNSTraverse traverses the DNS to show statistics and information
+#    Copyright (C) 2008 James Ponder
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, version 3 of the License.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module DNSTraverse
+  class ResolveError < RuntimeError
+  end
+  
+  module MessageUtility
+    module_function
+    
+    def msg_comment(msg, args)
+      warnings = Array.new
+      if args[:want_recursion] then
+        if not msg.header.ra then
+          warnings.push "#{msg.answerfrom} doesn't allow recursion"
+        end
+      else
+        if msg.header.ra then
+          warnings.push "#{msg.answerfrom} allows recursion"
+        end
+      end
+      if msg.header.tc then
+        warnings.push "#{msg.answerfrom} sent truncated packet"
+      end
+      for warn in warnings do
+        Log.warn { warn }
+      end
+      Log.debug { "#{msg.answerfrom} code #{msg.rcode}" }
+      return warnings
+    end
+    
+    def msg_validate(msg, args)
+      a = args.dup
+      a[:qclass]||= 'IN'
+      return true if msg.rcode != Dnsruby::RCode.NOERROR
+      begin
+        if msg.question.size != 1 then
+          raise ResolveError, "#{msg.answerfrom} returned unexpected " +
+          "question size #{msg.question.size}"
+        end
+        for c in [:qname, :qclass, :qtype] do
+          if a[c] and
+            a[c].to_s.downcase != msg.question[0].send(c).to_s.downcase then
+            raise ResolveError, "#{msg.answerfrom} returned mismatched #{c} " +
+          "#{msg.question[0].send(c)} instead of expected #{a[c]}"
+          end
+        end
+      rescue => e
+        Log.debug { "Failed message was: " + msg.to_s }
+        raise e
+      end
+      return true
+    end
+    
+    def msg_answers?(msg, args)
+      qname = args[:qname].to_s
+      qclass = (args[:qclass] || 'IN').to_s
+      qtype = args[:qtype].to_s
+      any = qtype.casecmp('ANY') == 0 ? true : false
+      ans = msg.answer.select { |x|
+        x.name.to_s.casecmp(qname) == 0 && 
+        x.klass.to_s.casecmp(qclass) == 0 &&
+         (any || x.type.to_s.casecmp(qtype) == 0)
+      }
+      Log.debug { "Answers:" + ans.size.to_s}
+      return ans.size > 0 ? ans : nil
+    end
+    
+    def msg_additional?(msg, args)
+      qclass = args[:qclass] || 'IN'
+      Log.debug { "Looking for #{args[:qname]}/#{args[:qtype]} in additional" }
+      add = msg.additional.select { |x|
+        x.name.to_s.casecmp(args[:qname].to_s) == 0 && 
+        x.klass.to_s.casecmp(qclass.to_s) == 0 &&
+        x.type.to_s.casecmp(args[:qtype].to_s) == 0
+      }
+      Log.debug { add.size > 0 ? "Found #{add.size} additional records" \
+        : "No additional records for #{args[:qname]}/#{args[:qtype]}"}
+      return add.size > 0 ? add : nil
+    end
+    
+    def msg_additional_ips?(msg, args)
+      qclass = args[:qclass] || 'IN'
+      Log.debug { "Looking for #{args[:qname]}/#{args[:qtype]} in additional" }
+      if add = msg.additional.select { |x|
+          x.name.to_s.casecmp(args[:qname].to_s) == 0 && 
+          x.klass.to_s.casecmp(qclass.to_s) == 0 &&
+          x.type.to_s.casecmp(args[:qtype].to_s) == 0
+        } then
+        ips = add.map {|x| x.address.to_s }
+        Log.debug { "Found in additional #{args[:qname]} = " + ips.join(", ") }
+        return ips
+      end
+      Log.debug { "No additional records for #{args[:qname]}/#{args[:qtype]}" }
+      return nil
+    end
+    
+    #    def msg_referrals(msg, args)
+    #      r = msg.authority.select { |x|
+    #        x.type.to_s.casecmp('NS') == 0 && x.klass.to_s.casecmp('IN') == 0
+    #      }
+    #      if args[:bailiwick] then
+    #        b = args[:bailiwick]
+    #        r = r.select { |x|
+    #          zonename = x.name.to_s
+    #          if cond = zonename !~ /#{@b}$/i then
+    #            Log.debug { "Excluding lame referral #{b} to #{zonename}" }
+    #            raise "lame"
+    #          end
+    #          cond
+    #        }
+    #      end
+    #      Log.debug { "Referrals: " + r.map {|x| x.domainname.to_s }.join(", ") }
+    #      return r
+    #    end
+    
+    def msg_authority(msg)
+      ns = []
+      soa = []
+      other = []
+      for rr in msg.authority do
+        type = rr.type.to_s
+        klass = rr.klass.to_s
+        if type.casecmp('NS') == 0 && klass.casecmp('IN') == 0
+          ns.push rr
+        elsif type.casecmp('SOA') == 0 && klass.casecmp('IN') == 0
+          soa.push rr
+        else
+          other.push rr
+        end
+      end
+      return ns, soa, other      
+    end
+    
+    def msg_follow_cnames(msg, args)
+      name = args[:qname]
+      type = args[:qtype]
+      bw = args[:bailiwick].to_s
+      bwend = ".#{args[:bailiwick]}"
+      while true do
+        return name if msg_answers?(msg, :qname => name, :qtype => type)
+        if not ans = msg_answers?(msg, :qname => name, :qtype => 'CNAME') then
+          return name
+        end
+        target = ans[0].domainname.to_s
+        Log.debug { "CNAME encountered from #{name} to #{target}"}
+        if bw and (target.casecmp(bw) != 0 and name !~ /#{bwend}$/i) then
+          # target outside of bailiwick, don't follow any more CNAMEs.
+          return target
+        end
+        name = target
+      end
+    end
+    
+    def msg_nodata?(msg)
+      ns, soa, other = msg_authority(msg)
+      if soa.size > 0 or ns.size == 0 then
+        Log.debug { "NODATA: soa=#{soa.size} ns=#{ns.size}" }
+        return true
+      end
+      return false
+    end
+    
+    def msg_cacheable(msg, bailiwick, type = :both)
+      good, bad = Array.new, Array.new
+      bw = bailiwick.to_s
+      bwend = "." + bw
+      for section in [:additional, :authority] do
+        for rr in msg.send(section) do
+          name = rr.name.to_s
+          if bailiwick.nil? or name.casecmp(bw) == 0 or
+            name =~ /#{bwend}$/i then
+            good.push rr
+          else
+            bad.push rr
+          end
+        end
+      end
+      good.map {|x| Log.debug { "Records within bailiwick: " + x.to_s } }
+      bad.map {|x| Log.debug { "Records outside bailiwick: " + x.to_s } }
+      return good if type == :good
+      return bad if type == :bad
+      return good, bad
+    end
+    
+  end
+end

data/lib/dnstraverse/referral.rb

@@ -0,0 +1,463 @@
+#    DNSTraverse traverses the DNS to show statistics and information
+#    Copyright (C) 2008 James Ponder
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, version 3 of the License.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'dnstraverse/response'
+require 'dnstraverse/response_noglue'
+require 'dnstraverse/info_cache'
+require 'dnstraverse/decoded_query_cache'
+
+module DNSTraverse
+  
+  class Referral
+    include MessageUtility
+    
+    attr_reader :server, :serverips, :qname, :qclass, :qtype, :nsatype
+    attr_reader :refid, :message, :infocache, :parent, :bailiwick, :stats
+    attr_reader :warnings, :children, :parent_ip
+    attr_reader :decoded_query_cache
+    
+    EMPTY_ARRAY = [].freeze
+    
+    def txt_ips_verbose
+      return '' unless @serverips
+      a = @serverips.map do |ip|
+        sprintf("%.1f%%=", 100 * @serverweights[ip]).concat(ip =~ /^key:([^:]+(:[^:]*)?)/ ? $1 : ip)
+      end
+      a.sort.join(',')
+    end
+    
+    def txt_ips
+      return '' unless @serverips
+      @serverips.map { |ip|
+        ip =~ /^key:/ ? @stats_resolve[ip][:response].to_s : ip
+      }.sort.join(',')
+    end
+    
+    # ips_as_array will return any IP addresses we know for this referral server
+    def ips_as_array
+      return EMPTY_ARRAY unless @serverips
+      my_ips = []
+      for ip in @serverips do
+        my_ips << ip unless ip =~ /^key:/
+      end
+      return my_ips
+    end
+    
+    def to_s
+      return "#{@refid} [#{@qname}/#{@qclass}/#{@qtype}] server=#{@server} " +
+      "server_ips=#{txt_ips()} bailiwick=#{@bailiwick}"
+    end
+    
+    def referral_resolution?
+      return @referral_resolution ? true : false
+    end
+    
+    # Referral object represents a particular referral to a specified server
+    # with given qname, qclass and qtype.
+    #
+    # roots can be passed in, which will be used to populate root hints in to
+    # the infocache, which if not passed in will be automatically created
+    #
+    # server can be nil which is a special case and causes all the roots
+    # to be added as child referrals (uses infocache to lookup roots)
+    #
+    # if the server's IP address(es) are known, they are passed in as serverips
+    # otherwise, we will resolve the serverips
+    #
+    # referral_resolution should be set to false.  children that are a result
+    # of a resolution of a referral that didn't have glue records will have
+    # this set to true so that you can distringuish this detail
+    def initialize(args)
+      @resolver = args[:resolver] # Dnsruby::Resolver object
+      @qname = args[:qname]
+      @qclass = args[:qclass] || :IN
+      @qtype = args[:qtype] || :A
+      @nsatype = args[:nsatype] || :A
+      @infocache = args[:infocache] || DNSTraverse::InfoCache.new
+      @roots = args[:roots]
+      @resolves = nil # Array of referral objects for resolving phase
+      @refid = args[:refid] || ''
+      @server = args[:server] || nil # nil for the root-root server
+      @serverips = args[:serverips] || nil
+      @responses = Hash.new # responses/exception for each IP in @serverips
+      @children = Hash.new # Array of child Referrer objects keyed by IP
+      @bailiwick = args[:bailiwick] || nil
+      @secure = args[:secure] || true # ensure bailiwick checks
+      @parent = args[:parent] || nil # Parent Referral
+      @parent_ip = args[:parent_ip] || nil # Parent Referral IP if applicable
+      @maxdepth = args[:maxdepth] || 10 # maximum depth before error
+      @decoded_query_cache = args[:decoded_query_cache]
+      @referral_resolution = args[:referral_resolution] || false # flag
+      @stats = nil # will contain statistics for answers
+      @stats_resolve = nil # will contain statistics for our resolve (if applic)
+      @serverweights = Hash.new # key is IP
+      @warnings = Array.new # warnings will be placed here
+      @processed = false # flag for processed? method
+      raise "Must pass Resolver" unless @resolver
+      @infocache.add_hints('', args[:roots]) if args[:roots] # add root hints
+      unless @decoded_query_cache then
+        dcq_args = { :resolver => @resolver}
+        @decoded_query_cache = DNSTraverse::DecodedQueryCache.new(dcq_args)
+      end
+      if serverips then # we know the server weights - we're not resolving
+        for ip in serverips do
+          @serverweights[ip] = 1.0 / @serverips.length
+        end
+      end
+      Log.debug { "New resolver object created: " + self.to_s }
+    end
+    
+    def showstats
+      s = Hash.new
+      ObjectSpace.each_object do |o|
+        s[o.class]||= 0
+        s[o.class]= s[o.class] + 1
+      end
+      s.sort {|a,b| a[1] <=> b[1]}.each do | c |
+        puts "#{c[1]} #{c[0]}"
+      end
+    end
+    
+    # clean up the workings
+    def cleanup(args = nil)
+      Log.debug { "cleaning: #{self}" }
+      @infocache = nil unless args and args[:infocache]
+      @cacheable_good = @cacheable_bad = nil unless args and args[:cacheable]
+      @starters = @starters_bailiwick = nil unless args and args[:starters]
+      @auth_ns = @auth_soa = @auth_other = nil unless args and args[:auth]
+      @children = nil unless args and args[:children]
+      @resolves = nil unless args and args[:resolves]
+      @responses = nil unless args and args[:responses]
+      @decoded_query_cache = nil unless args and args[:decoded_query_cache]
+      @resolver = nil unless args and args[:resolver]
+    end
+    
+    def inside_bailiwick?(name)
+      return true if @bailiwick.nil?
+      bwend = ".#{@bailiwick}"
+      namestr = name.to_s
+      return true if namestr.casecmp(@bailiwick) == 0
+      return true if namestr =~ /#{bwend}$/i
+      return false
+    end
+    
+    def noglue?
+      return false if @serverips
+      return false unless inside_bailiwick?(@server)
+      return true
+    end
+    
+    # resolve server to serverips, return list of Referral objects to process
+    def resolve(*args)
+      raise "This Referral object has already been resolved" if resolved?
+      if noglue? then
+        # foo.net IN NS ns.foo.net - no IP cached & no glue = failure
+        Log.debug { "Attempt to resolve #{@server} with a bailiwick referral " +
+                    " of #{bailiwick} - no glue record provided" }
+        return Array.new
+      end
+      refid = "#{@refid}.0"
+      child_refid = 1
+      starters, newbailiwick = @infocache.get_startservers(@server)
+      Log.debug { "Resolving #{@server} type #{@nsatype} " }
+      for starter in starters do
+        r = make_referral(:server => starter[:name],
+                          :serverips => starter[:ips],
+                          :referral_resolution => true,
+                          :qname => @server, :qclass => 'IN',
+        :qtype => @nsatype, :bailiwick => newbailiwick,
+        :refid => "#{refid}.#{child_refid}")
+         (@resolves||= []) << r
+        child_refid+= 1
+      end
+      # return a set of Referral objects that need to be processed
+      return @resolves
+    end
+    
+    def resolve_calculate
+      Log.debug { "Calculating resolution: #{self}" }
+      # create stats_resolve containing all the statistics of the resolution
+      @stats_resolve = Hash.new
+      if noglue? then # in-bailiwick referral without glue
+        r = DNSTraverse::Response::NoGlue.new(:qname => @qname,
+                                              :qclass => @qclass,
+                                              :qtype => @qtype,
+                                              :server => @server,
+                                              :ip => @parent_ip,
+                                              :bailiwick => @bailiwick)
+        @stats_resolve[r.stats_key] = { :prob => 1.0, :response => r,
+          :referral => self }
+      else
+        # normal resolve - combine children's statistics in to @stats_resolve
+        stats_calculate_children(@stats_resolve, @resolves, 1.0)
+      end
+      # now use this data to work out %age of each IP address returned
+      @serverweights = Hash.new
+      @stats_resolve.each_pair do |key, data|
+        # key = IP or key:blah, data is hash containing :prob, etc.
+        if data[:response].status == :answered then # RR records
+          # there were some answers - so add the probabilities in
+          answers = data[:response].answers # weight RRs evenly
+          for rr in answers do
+            @serverweights[rr.address.to_s]||= 0
+            @serverweights[rr.address.to_s]+= data[:prob] / answers.length
+          end
+        else
+          # there were no answers - use the special key and record probabilities
+          @serverweights[key]||= 0
+          @serverweights[key]+= data[:prob]
+        end
+      end
+      @serverips = @serverweights.keys
+      Log.debug { "Calculating resolution (answer): #{@serverips.join(',')}" }
+    end
+    
+    def stats_calculate_children(stats, children, weight)
+      percent = (1.0 / children.length) * weight
+      for child in children do
+        child.stats.each_pair do |key, data|
+          if not stats[key] then
+            # just copy the child's statistics for this key
+            stats[key] = data.dup
+            stats[key][:prob]*= percent
+          else
+            stats[key][:prob]+= data[:prob] * percent
+          end
+        end
+      end
+    end
+    
+    def answer_calculate
+      Log.debug { "Calculating answer: #{self}" }
+      @stats = Hash.new
+      
+      if not @server then
+        # special case - rootroot, no actual IPs, just root referrals
+        stats_calculate_children(@stats, @children[:rootroot], 1.0)
+        @stats.each_pair do |key, data|
+          Log.debug { sprintf "Answer: %.2f%% %s\n", data[:prob] * 100, key }
+        end
+        return
+      end
+      for ip in @serverips do
+        serverweight = @serverweights[ip] # set at initialize or at resolve
+        if ip =~ /^key:/ then # resolve failed for some reason
+          # pull out the statistics on the resolution and copy over
+          raise "duplicate key found" if @stats[ip] # assertion
+          if @stats_resolve[ip][:prob] != serverweight then # assertion
+            $stderr.puts "#{@stats_resolve[ip][:prob]} vs #{serverweight}"
+            @stats_resolve[ip].each_pair do |a,b|
+              $stderr.puts a
+            end
+            raise "unexpected probability" 
+          end
+          @stats[ip] = @stats_resolve[ip].dup
+          next
+        end
+        if @children[ip] then
+          stats_calculate_children(@stats, @children[ip], serverweight)
+        else
+          response = @responses[ip]
+          @stats[response.stats_key] = { :prob => serverweight,
+            :response => response, :referral => self }
+        end
+      end
+      @stats.each_pair do |key, data|
+        Log.debug { sprintf "Answer: %.2f%% %s\n", data[:prob] * 100, key }
+      end
+    end
+    
+    def processed?
+      return @processed
+    end
+    
+    def resolved?
+      # root-root is always resolved, otherwise check we have IP addresses
+      return true if is_rootroot?
+      return true if @noglue
+      return false if @serverips.nil?
+      return true
+    end
+    
+    def is_rootroot?
+      # rootroot is the topmost object representing an automatic referral
+      # to all the root servers
+      @server.nil? ? true : false
+    end
+    
+    def process(args)
+      raise "This Referral object has already been processed" if processed?
+      raise "You need to resolve this Referral object" unless resolved?
+      if (server) then
+        process_normal(args)
+      else
+        # special case - no server means start from the top with the roots
+        process_add_roots(args)
+      end
+      # return a set of Referral objects that need to be processed
+      # XXX flatten really necessary?
+      @processed = true
+      return @children.values.flatten.select {|x| x.is_a? Referral}
+    end
+    
+    def process_add_roots(args)
+      Log.debug { "Special case processing, addding roots as referrals" }
+      refid_prefix = @refid == '' ? '' : "#{@refid}."
+      refid = 1
+      starters = (@infocache.get_startservers('', @nsatype))[0]
+      @children[:rootroot] = Array.new # use 'rootroot' instead of IP address
+      for root in starters do
+        r = make_referral(:server => root[:name], :serverips => root[:ips],
+                          :refid => "#{refid_prefix}#{refid}")
+        @children[:rootroot].push r
+        refid+= 1
+      end
+    end
+    
+    def check_loop?(args) # :ip, :qtype, :qname, :qclass
+      parent = @parent
+      until parent.nil? do
+        if parent.qname.to_s == args[:qname].to_s and
+          parent.qclass.to_s == args[:qclass].to_s and
+          parent.qtype.to_s == args[:qtype].to_s and
+          parent.serverips and parent.serverips.include?(args[:ip]) then
+          exit 1 # XXX fix me
+          return RuntimeError.new("Loop detected")
+        end
+        parent = parent.parent
+      end
+      return nil
+    end
+    
+    def process_normal(args)
+      Log.debug { "process " + self.to_s }
+      #      if l = check_loop?(:ip => ip, :qname => @qname,
+      #                         :qtype => @qtype, :qclass => @qclass) then
+      #        for ip in @serverips do
+      #          @responses[ip] = l
+      #          done
+      #          return
+      #        end
+      #      end
+      for ip in @serverips do
+        Log.debug { "Process normal #{ip}" }
+        next if ip =~ /^key:/ # resolve failed on something
+        m = nil
+        if @refid.scan(/\./).length >= @maxdepth.to_i then
+          m = RuntimeError.new "Maxdepth #{@maxdepth} exceeded"
+        end
+        Log.debug { "Process normal #{ip} - making response" }
+        r = DNSTraverse::Response.new(:message => m, :qname => @qname,
+                                      :qclass => @qclass, :qtype => @qtype,
+                                      :bailiwick => @bailiwick,
+                                      :infocache => @infocache, :ip => ip,
+                                      :decoded_query_cache => @decoded_query_cache)
+        Log.debug { "Process normal #{ip} - done making response" }
+        @responses[ip] = r
+        case r.status
+          when :restart, :referral then
+          Log.debug { "Process normal #{ip} - making referrals" }
+          @children[ip] = make_referrals(:qname => r.endname,
+                                         :starters => r.starters,
+                                         :bailiwick => r.starters_bailiwick,
+                                         :infocache => r.infocache,
+                                         :parent_ip => ip)
+          Log.debug { "Process normal #{ip} - done making referrals" }
+          # XXX shouldn't be any children unless referrals
+          #when :referral_lame then
+          #@children[ip] = RuntimeError.new "Improper or lame delegation"
+        end
+      end
+    end
+    
+    def make_referrals(args) # :starters can be @root or our own list
+      starters = args[:starters]
+      children = Array.new
+      child_refid = 1
+      for starter in starters do
+        refargs = {
+          :server => starter[:name],
+          :serverips => starter[:ips],
+          :refid => "#{refid}.#{child_refid}"
+        }.merge(args)
+        children.push make_referral(refargs)
+        child_refid+= 1
+      end
+      return children
+    end
+    
+    def make_referral(args)
+      raise "Must pass new refid" unless args[:refid]
+      refargs = { :qname => @qname, :qclass => @qclass,
+        :qtype => @qtype, :nsatype => @nsatype, :infocache => @infocache,
+        :referral_resolution => @referral_resolution,
+        :resolver => @resolver, :maxdepth => @maxdepth, :parent => self,
+        :decoded_query_cache => @decoded_query_cache }.merge(args)
+      return Referral.new(refargs)
+    end
+    
+    def replace_child(before, after)
+      @children.each_key do | ip |
+        @children[ip].map! { |c| c.equal?(before) ? after : c }
+      end
+    end
+    
+    def stats_display(args)
+      spacing = args[:spacing] || false
+      results = args[:results] || true
+      prefix = args[:prefix] || ''
+      indent = args[:indent] || "#{prefix}            "
+      first = true
+      @stats.keys.sort!.each do | key |
+        data = @stats[key]
+        puts if spacing and not first
+        first = false
+        printf "#{prefix}%5.1f%%: ", data[:prob] * 100
+        response = data[:response]
+        referral = data[:referral]
+        where = "#{referral.server} (#{response.ip})"
+        case response.status
+          when :exception
+          puts "#{response.exception_message} at #{where}"
+          when :noglue
+          puts "No glue at #{referral.parent.server} " + 
+            "(#{response.ip}) for #{response.server}"
+          when :error
+          puts "#{response.error_message} at #{where}"
+          when :nodata
+          puts "NODATA (for this type) at #{where})"
+          when :answered
+          puts "Answer from #{where}"
+          if results then
+            for rr in data[:response].answers do
+              puts "#{indent}#{rr}"
+            end
+          end
+        else
+          puts "Stopped at #{where})"
+          puts "#{indent}#{key}"
+        end
+        if response.status != :answered and
+         ((response.qname != @qname) or (response.qclass != @qclass) or
+         (response.qtype != @qtype)) then
+          puts "#{indent}While querying #{response.qname}/" +
+          "#{response.qclass}/#{response.qtype}"
+        end
+      end
+    end
+    
+  end
+  
+end