em-dns 0.1.1

data/Rakefile

@@ -0,0 +1,21 @@
+begin
+  require 'jeweler'
+rescue LoadError
+  raise "Jeweler, or one of its dependencies, is not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end
+
+Jeweler::Tasks.new do |s|
+  s.name = "em-dns"
+  s.summary = "Resolve domain names from EventMachine natively"
+  s.email = "astro@spaceboyz.net"
+  s.homepage = "http://github.com/astro/em-dns"
+  s.description = "DNS::Resolv made ready for EventMachine"
+  s.authors = ["Aman Gupta", "Stephan Maka"]
+  s.files =  FileList["[A-Z]*", "{lib,test,examples}/**/*"]
+  s.add_dependency 'eventmachine'
+end
+
+require 'rake/testtask'
+Rake::TestTask.new do |t|
+  t.test_files = FileList["test/test*.rb"]
+end

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:patch: 1
+:major: 0
+:minor: 1

data/examples/lookup_many.rb

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+
+require 'uri'
+require 'eventmachine'
+$: << File.dirname(__FILE__) + '/../lib'
+require 'em/dns_resolver'
+
+if ARGV.size == 0
+  puts "Usage: #{$0} <url-files>"
+  exit
+end
+
+
+hosts = []
+ARGV.each do |fn|
+  IO::readlines(fn).each do |line|
+      line.chomp!
+      line.split(/\s+/).each do |url|
+        uri = URI::parse(url)
+        hosts << uri.host
+    end
+  end
+end
+hosts.uniq!
+puts "Will resolve #{hosts.size} hosts"
+
+BATCH_SIZE = 10
+successes = 0
+failures = 0
+t1 = Time.now
+EM.run {
+  pending = 0
+  EM.add_periodic_timer(0.1) do
+    batch, hosts = hosts[0..(BATCH_SIZE-1)], (hosts[BATCH_SIZE..-1] || [])
+    batch.each do |host|
+      df = EM::DnsResolver.resolve(host)
+      df.callback { |a|
+        p host => a
+        successes += 1
+        pending -= 1
+        EM.stop if pending < 1 && hosts.empty?
+      }
+      df.errback { |*a|
+        puts "Cannot resolve #{host}: #{a.inspect}"
+        failures += 1
+        pending -= 1
+        EM.stop if pending < 1 && hosts.empty?
+      }
+      pending += 1
+    end
+    puts "#{pending} pending"
+  end
+}
+t2 = Time.now
+puts "#{successes} successful, #{failures} failures in #{t2 - t1} s"

data/lib/em/dns_cache.rb

@@ -0,0 +1,347 @@
+# $Id: dns_cache.rb 5040 2007-10-05 17:31:04Z francis $
+#
+#
+
+require 'rubygems'
+require 'eventmachine'
+require 'resolv'
+
+
+module EventMachine
+  module DnsCache
+
+    class Cache
+      def initialize
+        @hash = {}
+      end
+      def add domain, value, expiration
+        ex = ((expiration < 0) ? :none : (Time.now + expiration))
+        @hash[domain] = [ex, value]
+      end
+      def retrieve domain
+        if @hash.has_key?(domain)
+          d = @hash[domain]
+          if d.first != :none and d.first < Time.now
+            @hash.delete(domain)
+            nil
+          else
+            d.last
+          end
+        end
+      end
+    end
+
+    @a_cache = Cache.new
+    @mx_cache = Cache.new
+    @nameservers = []
+    @message_ix = 0
+    MAX_WAITING = 100
+    @waiting = 0
+    @pending = []
+
+    def self.add_nameserver ns
+      @nameservers << ns unless @nameservers.include?(ns)
+    end
+
+    def self.add_nameservers_from_file file='/etc/resolv.conf'
+      IO::readlines(file).each do |line|
+        if line =~ /^nameserver (.+)$/
+          $1.split(/\s+/).each { |ns|
+            @nameservers << ns unless ns.empty?
+          }
+        end
+      end
+    end
+
+    def self.verbose v=true
+      @verbose = v
+    end
+
+
+    def self.add_cache_entry cache_type, domain, value, expiration
+      cache = if cache_type == :mx
+                @mx_cache
+              elsif cache_type == :a
+                @a_cache
+              else
+                raise "bad cache type"
+              end
+
+      v = EM::DefaultDeferrable.new
+      v.succeed( value.dup.freeze )
+      cache.add domain, v, expiration
+    end
+
+    # Needs to be DRYed up with resolve_mx.
+    #
+    def self.resolve domain
+      if d = @a_cache.retrieve(domain)
+        puts "Cache hit for #{domain}" if @verbose
+        look_pending
+        d
+      else
+=begin
+        d = @a_cache[domain]
+        if d.first < Time.now
+          STDOUT.puts "Expiring stale cache entry for #{domain}" if @verbose
+          @a_cache.delete domain
+          resolve domain
+        else
+          STDOUT.puts "Fulfilled #{domain} from cache" if @verbose
+          d.last
+        end
+      else
+=end
+        if @waiting >= MAX_WAITING
+          puts "Postponing #{domain} because already waiting for #{@waiting} queries" if @verbose
+          d = EM::DefaultDeferrable.new
+          @pending << lambda {
+            d_inner = resolve domain
+            d_inner.callback &d.method(:succeed)
+            d_inner.errback &d.method(:fail)
+          }
+          puts "#{@pending.size} pending requests now" if @verbose
+          d
+        else
+          d = resolve_do domain
+          @waiting += 1
+          STDOUT.puts "Now waiting for #{@waiting}" if @verbose
+          on_one_done = lambda {
+            @waiting -= 1
+            look_pending
+          }
+          d.callback &on_one_done
+          d.errback &on_one_done
+          d
+        end
+      end
+    end
+
+    def self.look_pending
+      EM.next_tick {
+        while @waiting < MAX_WAITING && !@pending.empty?
+          pending1 = @pending.shift
+          pending1.call
+          puts "#{@pending.size} pending requests now" if @verbose
+        end
+      }
+    end
+
+    def self.resolve_do domain
+      STDOUT.puts "Fulfilling #{domain} from network" if @verbose
+      d = EM::DefaultDeferrable.new
+      #d.timeout(5)
+      d.callback { d.cancel_timeout }
+      d.errback { d.cancel_timeout }
+      @a_cache.add domain, d, 300 # Hard-code a 5 minute expiration
+      #@a_cache[domain] = [Time.now+120, d] # Hard code a 120-second expiration.
+
+      lazy_initialize
+      m = Resolv::DNS::Message.new
+      m.rd = 1
+      m.add_question domain, Resolv::DNS::Resource::IN::A
+      m = m.encode
+      d_inner = EM::DefaultDeferrable.new
+      @nameservers.each {|ns|
+        @message_ix = (@message_ix + 1) % 60000
+        Request.new d_inner, @message_ix
+        msg = m.dup
+        msg[0,2] = [@message_ix].pack("n")
+        @u.send_datagram msg, ns, 53
+      }
+
+      d_inner.callback {|resp|
+        r = []
+        resp.each_answer {|name,ttl,data|
+          r << data.address.to_s if data.kind_of?(Resolv::DNS::Resource::IN::A)
+        }
+
+        # Freeze the array since we'll be keeping it in cache and passing it
+        # around to multiple users. And alternative would have been to dup it.
+        r.freeze
+        puts "Succeeding with #{r.inspect}"
+        d.succeed r
+      }
+      d_inner.errback &d.method(:fail)
+
+
+      d
+    end
+
+
+    # Needs to be DRYed up with resolve.
+    #
+    def self.resolve_mx domain
+      if d = @mx_cache.retrieve(domain)
+        d
+      else
+=begin
+           if @mx_cache.has_key?(domain)
+             d = @mx_cache[domain]
+             if d.first < Time.now
+               STDOUT.puts "Expiring stale cache entry for #{domain}" if @verbose
+               @mx_cache.delete domain
+               resolve_mx domain
+             else
+               STDOUT.puts "Fulfilled #{domain} from cache" if @verbose
+               d.last
+             end
+           else
+=end
+        STDOUT.puts "Fulfilling #{domain} from network" if @verbose
+        d = EM::DefaultDeferrable.new
+        d.timeout(5)
+        #@mx_cache[domain] = [Time.now+120, d] # Hard code a 120-second expiration.
+        @mx_cache.add domain, d, 300 # Hard-code a 5 minute expiration
+
+        mx_query = MxQuery.new d
+
+        lazy_initialize
+        m = Resolv::DNS::Message.new
+        m.rd = 1
+        m.add_question domain, Resolv::DNS::Resource::IN::MX
+        m = m.encode
+        @nameservers.each {|ns|
+          @message_ix = (@message_ix + 1) % 60000
+          Request.new mx_query, @message_ix
+          msg = m.dup
+          msg[0,2] = [@message_ix].pack("n")
+          @u.send_datagram msg, ns, 53
+        }
+
+
+        d
+      end
+    end
+
+
+    def self.lazy_initialize
+      # Will throw an exception if EM is not running.
+      # We wire a signaller into the socket handler to tell us when that socket
+      # goes away. (Which can happen, among other things, if the reactor
+      # stops and restarts.)
+      #
+      raise "EventMachine reactor not running" unless EM.reactor_running?
+
+      unless @u
+        us = proc {@u = nil}
+        @u = EM::open_datagram_socket( "0.0.0.0", 0, Socket ) {|c|
+          c.unbind_signaller = us
+        }
+      end
+
+    end
+
+
+    def self.parse_local_mx_records txt
+      domain = nil
+      addrs = []
+
+      add_it = proc {
+        a = addrs.sort {|m,n| m.last <=> n.last}.map {|y| y.first}
+        add_cache_entry :mx, domain, a, -1
+      }
+
+      txt = StringIO.new( txt ) if txt.is_a?(String)
+      txt.each_line {|ln|
+        if ln =~ /\A\s*([\d\w\.\-\_]+)\s+(\d+)\s*\Z/
+          if domain
+            addrs << [$1.dup, $2.dup.to_i]
+          end
+        elsif ln =~ /\A\s*([^\s\:]+)\s*\:\s*\Z/
+          add_it.call if domain
+          domain = $1.dup
+          addrs.clear
+        end
+      }
+
+      add_it.call if domain
+    end
+
+
+    class MxQuery
+      include EM::Deferrable
+
+      def initialize rslt
+        @result = rslt # Deferrable
+        @n_a_lookups = 0
+
+        self.callback {|resp|
+          addrs = {}
+          resp.each_additional {|name,ttl,data|
+            addrs.has_key?(name) ? (addrs[name] << data.address.to_s) : (addrs[name] = [data.address.to_s])
+          }
+
+          @addresses = resp.answer.
+          sort {|a,b| a[2].preference <=> b[2].preference}.
+          map {|name,ttl,data|
+            ex = data.exchange
+            addrs[ex] or EM::DnsCache.resolve(ex.to_s)
+          }
+
+          @addresses.each_with_index {|a,ix|
+            if a.respond_to?(:set_deferred_status)
+              @n_a_lookups += 1
+              a.callback {|r|
+                @addresses[ix] = r
+                @n_a_lookups -= 1
+                succeed_result if @n_a_lookups == 0
+              }
+            end
+          }
+
+          succeed_result if @n_a_lookups == 0
+        }
+      end
+
+      def succeed_result
+        # Questionable whether we should uniq if it perturbs the sort order.
+        # Also freeze it so some user can't wipe it out on us.
+        @result.succeed @addresses.flatten.uniq.freeze
+      end
+
+    end
+
+    class Request
+      include EM::Deferrable
+
+      @@outstanding = {}
+
+      def self.post response
+        if r = @@outstanding.delete(response.id)
+          r.succeed response
+        end
+      end
+
+      def initialize rslt, m_id
+        @result = rslt
+        @msgid = m_id
+        raise "request-queue overflow" if @@outstanding.has_key?(@msgid)
+        @@outstanding[@msgid] = self
+
+        self.timeout(10)
+        self.errback { self.cancel_timeout; @@outstanding.delete(@msgid); @result.fail }
+        self.callback {|resp| self.cancel_timeout; @result.succeed resp }
+      end
+    end
+
+    class Socket < EM::Connection
+      attr_accessor :unbind_signaller
+
+      def receive_data dg
+        m = nil
+        begin
+          m = Resolv::DNS::Message.decode dg
+        rescue
+        end
+        Request.post(m) if m
+      end
+
+      def unbind
+        @unbind_signaller.call if @unbind_signaller
+      end
+    end
+
+  end
+end
+

data/lib/em/dns_resolver.rb

@@ -0,0 +1,156 @@
+require 'eventmachine'
+require 'resolv'
+
+module EventMachine
+  module DnsResolver
+    ##
+    # Global interface
+    ##
+
+    def self.resolve(hostname)
+      Request.new(socket, hostname)
+    end
+
+    def self.socket
+      unless defined?(@socket)
+        @socket = DnsSocket.open
+      end
+      @socket
+    end
+
+    def self.nameserver=(ns)
+      @nameserver = ns
+    end
+    def self.nameserver
+      unless defined?(@nameserver)
+        IO::readlines('/etc/resolv.conf').each do |line|
+          if line =~ /^nameserver (.+)$/
+            @nameserver = $1.split(/\s+/).first
+          end
+        end
+      end
+      @nameserver
+    end
+
+    ##
+    # Socket stuff
+    ##
+
+    class RequestIdAlreadyUsed < RuntimeError
+    end
+
+    class DnsSocket < EM::Connection
+      def self.open
+        EM::open_datagram_socket('0.0.0.0', 0, self)
+      end
+      def post_init
+        @requests = {}
+        EM.add_periodic_timer(0.1, &method(:tick))
+      end
+      # Periodically called each second to fire request retries
+      def tick
+        @requests.each do |id,req|
+          req.tick
+        end
+      end
+      def register_request(id, req)
+        if @requests.has_key?(id)
+          raise RequestIdAlreadyUsed
+        else
+          @requests[id] = req
+        end
+      end
+      def send_packet(pkt)
+        send_datagram(pkt, nameserver, 53)
+      end
+      def nameserver=(ns)
+        @nameserver = ns
+      end
+      def nameserver
+        @nameserver ||= DnsResolver.nameserver
+      end
+      # Decodes the packet, looks for the request and passes the
+      # response over to the requester
+      def receive_data(data)
+        msg = nil
+        begin
+          msg = Resolv::DNS::Message.decode data
+        rescue
+        else
+          req = @requests[msg.id]
+          if req
+            @requests.delete(msg.id)
+            req.receive_answer(msg)
+          end
+        end
+      end
+    end
+
+    ##
+    # Request
+    ##
+
+    class Request
+      include Deferrable
+      attr_accessor :retry_interval
+      attr_accessor :max_tries
+      def initialize(socket, hostname)
+        @socket = socket
+        @hostname = hostname
+        @tries = 0
+        @last_send = Time.at(0)
+        @retry_interval = 3
+        @max_tries = 5
+        EM.next_tick { tick }
+      end
+      def tick
+        # Break early if nothing to do
+        return if @last_send + @retry_interval > Time.now
+
+        if @tries < @max_tries
+          send
+        else
+          fail 'retries exceeded'
+        end
+      end
+      # Called by DnsSocket#receive_data
+      def receive_answer(msg)
+        addrs = []
+        msg.each_answer do |name,ttl,data|
+          if data.kind_of?(Resolv::DNS::Resource::IN::A) ||
+              data.kind_of?(Resolv::DNS::Resource::IN::AAAA)
+            addrs << data.address.to_s
+          end
+        end
+        if addrs.empty?
+          fail "rcode=#{msg.rcode}"
+        else
+          succeed addrs
+        end
+      end
+      private
+      def send
+        @socket.send_packet(packet.encode)
+        @tries += 1
+        @last_send = Time.now
+      end
+      def id
+        begin
+          @id = rand(65535)
+          @socket.register_request(@id, self)
+        rescue RequestIdAlreadyUsed
+          retry
+        end unless defined?(@id)
+
+        @id
+      end
+      def packet
+        msg = Resolv::DNS::Message.new
+        msg.id = id
+        msg.rd = 1
+        msg.add_question @hostname, Resolv::DNS::Resource::IN::A
+        msg
+      end
+    end
+  end
+end

data/test/test_basic.rb

@@ -0,0 +1,222 @@
+# $Id: test_basic.rb 5040 2007-10-05 17:31:04Z francis $
+#
+#
+
+require 'test/unit'
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+require 'em/dns_cache'
+
+
+
+
+#--------------------------------------
+
+
+class TestBasic < Test::Unit::TestCase
+
+	TestNameserver = "151.202.0.85"
+	TestNameserver2 = "151.202.0.86"
+
+	LocalMxRecords = %Q(
+	boondoggle.zzz:
+	65.66.67.68 10
+	55.56.57.58 5
+	esmtp.someone.zzz 4
+
+	boondoggle.yyy:
+	)
+
+	def test_a
+		EM::DnsCache.add_nameservers_from_file
+		EM::DnsCache.verbose
+
+		out = nil
+
+		EM.run {
+			d = EM::DnsCache.resolve "bayshorenetworks.com"
+			d.errback {EM.stop}
+			d.callback {|r|
+				d = EM::DnsCache.resolve "bayshorenetworks.com"
+				d.errback { EM.stop }
+				d.callback {|r| puts r; out = r; EM.stop }
+			}
+		}
+
+		assert out
+	end
+
+	def test_a_pair
+		EM::DnsCache.add_nameservers_from_file
+		EM::DnsCache.verbose
+
+		out = nil
+
+		EM.run {
+			d = EM::DnsCache.resolve "maila.microsoft.com"
+			d.errback {EM.stop}
+			d.callback {|r|
+				out = r
+				EM.stop
+			}
+		}
+
+		assert_equal( Array, out.class )
+		assert_equal( 2, out.length )
+	end
+
+
+	# This test causes each request to hit the network because they're all scheduled
+	# before the first one can come back and load the cache. Although a nice mis-feature for
+	# stress testing, it would be nice to fix it someday, perhaps by not kicking off a
+	# request for a particular domain if one is already pending.
+	# Without epoll, this test gets really slow and usually doesn't complete.
+	def test_lots_of_a
+		EM.epoll
+		EM::DnsCache.add_nameserver TestNameserver
+		EM::DnsCache.add_nameserver TestNameserver2
+		EM::DnsCache.verbose 
+
+		n = 250
+		e = 0
+		s = 0
+		EM.run {
+			n.times {
+				d = EM::DnsCache.resolve "ibm.com"
+				d.errback {e+=1; n -= 1; EM.stop if n == 0}
+				d.callback {s+=1; n -= 1; EM.stop if n == 0}
+			}
+		}
+		assert_equal( 0, n)
+		assert_equal( 250, s)
+	end
+
+
+
+
+	def test_mx
+		EM::DnsCache.add_nameserver TestNameserver
+		EM::DnsCache.verbose
+
+		out = nil
+
+		EM.run {
+			d = EM::DnsCache.resolve_mx "steamheat.net"
+			d.errback {EM.stop}
+			d.callback {|r|
+				p r
+				d = EM::DnsCache.resolve_mx "steamheat.net"
+				d.errback {EM.stop}
+				d.callback {|r|
+					out = r
+					p r
+					EM.stop
+				}
+			}
+		}
+
+		assert out
+	end
+
+
+	# The arrays of addresses we get back from the DnsCache are FROZEN.
+	# That's because the same objects get passed around to any caller that
+	# asks for them. If you need to modify the array, dup it.
+	#
+	def test_freeze
+		EM::DnsCache.add_nameserver TestNameserver
+		EM::DnsCache.verbose
+
+		out = nil
+
+		EM.run {
+			d = EM::DnsCache.resolve_mx "steamheat.net"
+			d.errback {EM.stop}
+			d.callback {|r|
+				out = r
+				EM.stop
+			}
+		}
+
+		assert out
+		assert( out.length > 0)
+		assert_raise( TypeError ) {
+			out.clear
+		}
+	end
+
+
+	def test_local_defs
+		EM::DnsCache.add_nameserver TestNameserver
+		EM::DnsCache.verbose
+
+		EM::DnsCache.add_cache_entry( :mx, "example.zzz", ["1.2.3.4"], -1 )
+		out = nil
+		EM.run {
+			d = EM::DnsCache.resolve_mx "example.zzz"
+			d.errback {EM.stop}
+			d.callback {|r|
+				out = r
+				EM.stop
+			}
+		}
+		assert_equal( ["1.2.3.4"], out )
+	end
+
+	def test_multiple_local_defs
+		EM::DnsCache.verbose
+
+		EM::DnsCache.add_cache_entry( :mx, "example.zzz", ["1.2.3.4", "5.6.7.8"], -1 )
+		out = nil
+		EM.run {
+			d = EM::DnsCache.resolve_mx "example.zzz"
+			d.errback {EM.stop}
+			d.callback {|r|
+				out = r
+				EM.stop
+			}
+		}
+		assert_equal( ["1.2.3.4","5.6.7.8"], out )
+	end
+
+	# Adding cache entries where they already exist causes them to be REPLACED.
+	#
+	def test_replace
+		EM::DnsCache.verbose
+
+		EM::DnsCache.add_cache_entry( :mx, "example.zzz", ["1.2.3.4", "5.6.7.8"], -1 )
+		EM::DnsCache.add_cache_entry( :mx, "example.zzz", ["10.11.12.13"], -1 )
+		out = nil
+		EM.run {
+			d = EM::DnsCache.resolve_mx "example.zzz"
+			d.errback {EM.stop}
+			d.callback {|r|
+				out = r
+				EM.stop
+			}
+		}
+		assert_equal( ["10.11.12.13"], out )
+	end
+
+	# We have a facility for storing locally-defined MX records.
+	# The DNS cache has a way to parse and process these values.
+	#
+	def test_external_mx_defs
+		EM::DnsCache.verbose
+
+		EM::DnsCache.parse_local_mx_records LocalMxRecords
+
+		out = nil
+		EM.run {
+			d = EM::DnsCache.resolve_mx "boondoggle.zzz"
+			d.errback {EM.stop}
+			d.callback {|r|
+				out = r
+				EM.stop
+			}
+		}
+		assert_equal( ["esmtp.someone.zzz", "55.56.57.58", "65.66.67.68"], out )
+	end
+
+end
+
+

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification 
+name: em-dns
+version: !ruby/object:Gem::Version 
+  version: 0.1.1
+platform: ruby
+authors: 
+- Aman Gupta
+- Stephan Maka
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-11-07 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: DNS::Resolv made ready for EventMachine
+email: astro@spaceboyz.net
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- Rakefile
+- VERSION.yml
+- examples/lookup_many.rb
+- lib/em/dns_cache.rb
+- lib/em/dns_resolver.rb
+- test/test_basic.rb
+has_rdoc: true
+homepage: http://github.com/astro/em-dns
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Resolve domain names from EventMachine natively
+test_files: 
+- test/test_basic.rb
+- examples/lookup_many.rb