passivedns-client 2.1.6 → 2.1.12

data/lib/passivedns/client/provider/bfk.rb

@@ -5,7 +5,7 @@ module PassiveDNS #:nodoc: don't document this
   module Provider
 
     # Queries BFK.de's passive DNS database
-  	class BFK < PassiveDB
+    class BFK < PassiveDB
       # Sets the modules self-reported name to "BFK.de"
       def self.name
         "BFK.de"
@@ -34,72 +34,76 @@ module PassiveDNS #:nodoc: don't document this
       #
       #   PassiveDNS::Provider::BFK.new(options)
       #
-      
-  		def initialize(options={})
-  			@debug = options[:debug] || false
+      def initialize(options={})
+        @debug = options[:debug] || false
+        @timeout = options[:timeout] || 20
         @base = options["URL"] || "http://www.bfk.de/bfk_dnslogger.html?query="
-  		end
+        raise "Due to the EU GDPR policy, this service has been shut down until further notice."
+      end
     
       # Takes a label (either a domain or an IP address) and returns
       # an array of PassiveDNS::PDNSResult instances with the answers to the query
-   		def lookup(label, limit=nil)	
-  			$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
-  			Timeout::timeout(240) {
-  				t1 = Time.now
-  				open(
-  					@base+label,
-  					"User-Agent" => "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}"
-  				) do |f|
-  					t2 = Time.now
-  					recs = parse(f.read,t2-t1)
+      def lookup(label, limit=nil)  
+        $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
+        Timeout::timeout(@timeout) {
+          t1 = Time.now
+          open(
+            @base+label,
+            "User-Agent" => "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}"
+          ) do |f|
+            t2 = Time.now
+            recs = parse(f.read,t2-t1)
             if limit
               recs[0,limit]
             else
               recs
             end
-  				end
-  			}
-  		rescue Timeout::Error => e
-  			$stderr.puts "#{self.class.name} lookup timed out: #{label}"      
-  		end
+          end
+        }
+      rescue Timeout::Error
+        $stderr.puts "#{self.class.name} lookup timed out: #{label}"      
+      end
     
       private
     
       # parses the webpage returned by BFK to generate an array of PDNSResult
-  		def parse(page,response_time)
-  			line = page.unpack('C*').pack('U*').split(/<table/).grep(/ id=\"logger\"/)
-  			return [] unless line.length > 0
-  			line = line[0].gsub(/[\t\n]/,'').gsub(/<\/table.*/,'')
-  			rows = line.split(/<tr.*?>/)
-  			res = []
-  			rows.collect do |row|
-  				r = row.split(/<td>/).map{|x| x.gsub(/<.*?>/,'').gsub(/\&.*?;/,'')}[1,1000]
-  				if r and r[0] =~ /\w/
-  					# BFK includes the MX weight in the answer response. First, find the MX records, then dump the weight to present a consistent record name to the collecting array. Otherwise the other repositories will present the same answer and your results will become cluttered with duplicates.
-  					if r[1] == "MX" then
-  						# MX lines look like "5 mx.domain.tld", so split on the space and assign r[2] (:answer) to the latter part.
-  						#s = r[2].split(/\w/).map{|x| x}[1,1000]
-  						#	r[2] = s[1]
-  						r[2] =~ /[0-9]+?\s(.+)/
-  						s=$1
-  						puts "DEBUG: == BFK: MX Parsing Strip: Answer: " + r[2] + " : mod: " + s if @debug
-  						r[2] = s
-						
-  							######### FIX BLANKS FOR MX
-						
-  					end
+      def parse(page,response_time)
+        line = page.unpack('C*').pack('U*').split(/<table/).grep(/ id=\"logger\"/)
+        return [] unless line.length > 0
+        line = line[0].gsub(/[\t\n]/,'').gsub(/<\/table.*/,'')
+        rows = line.split(/<tr.*?>/)
+        res = []
+        rows.collect do |row|
+          r = row.split(/<td>/).map{|x| x.gsub(/<.*?>/,'').gsub(/\&.*?;/,'')}[1,1000]
+          if r and r[0] =~ /\w/
+            # BFK includes the MX weight in the answer response. First, find the MX records, 
+            #    then dump the weight to present a consistent record name to the collecting 
+            #    array. Otherwise the other repositories will present the same answer and 
+            #    your results will become cluttered with duplicates.
+            if r[1] == "MX" then
+              # MX lines look like "5 mx.domain.tld", so split on the space and assign r[2] (:answer) to the latter part.
+              #s = r[2].split(/\w/).map{|x| x}[1,1000]
+              #  r[2] = s[1]
+              r[2] =~ /[0-9]+?\s(.+)/
+              s=$1
+              #puts "DEBUG: == BFK: MX Parsing Strip: Answer: " + r[2] + " : mod: " + s if @debug
+              r[2] = s
+            
+                ######### FIX BLANKS FOR MX
+            
+            end
             query = r[0]
             answer = r[2]
             rrtype = r[1]
-  					res << PDNSResult.new(self.class.name,response_time,query,answer,rrtype)
-  				end
-  			end
-  			res
-  		rescue Exception => e
-  			$stderr.puts "#{self.class.name} Exception: #{e}"
-  			raise e
-  		end
+            res << PDNSResult.new(self.class.name,response_time,query,answer,rrtype,'white')
+          end
+        end
+        res
+      rescue Exception => e
+        $stderr.puts "#{self.class.name} Exception: #{e}"
+        raise e
+      end
 
-  	end
+    end
   end
-end
+end

data/lib/passivedns/client/provider/circl.rb

@@ -9,7 +9,7 @@ module PassiveDNS #:nodoc: don't document this
   module Provider
     # Queries CIRCL.LU's passive DNS database
     # Circl is aliased by CIRCL
-  	class Circl < PassiveDB
+    class Circl < PassiveDB
       # Sets the modules self-reported name to "CIRCL"
       def self.name
         "CIRCL"
@@ -45,68 +45,84 @@ module PassiveDNS #:nodoc: don't document this
       #
       #   PassiveDNS::Provider::CIRCL.new(options)
       #
-  		def initialize(options={})
+      def initialize(options={})
         @debug = options[:debug] || false
+        @timeout = options[:timeout] || 20
         @username = options["USERNAME"]
         @password = options["PASSWORD"]
         @auth_token = options["AUTH_TOKEN"]
         @url = options["URL"] || "https://www.circl.lu/pdns/query"
-  		end
+      end
 
       # Takes a label (either a domain or an IP address) and returns
       # an array of PassiveDNS::PDNSResult instances with the answers to the query
-  		def lookup(label, limit=nil)
-  			$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
-  			Timeout::timeout(240) {
-  				url = @url+"/"+label
-  				$stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
-  				url = URI.parse url
-  				http = Net::HTTP.new(url.host, url.port)
-  				http.use_ssl = (url.scheme == 'https')
-  				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-  				http.verify_depth = 5
-  				request = Net::HTTP::Get.new(url.request_uri)
-  				request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
+      def lookup(label, limit=nil)
+        $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
+        recs = []
+        Timeout::timeout(@timeout) {
+          url = @url+"/"+label
+          $stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
+          begin
+            url = URI.parse url
+          rescue URI::InvalidURIError
+            $stderr.puts "ERROR: Invalid address: #{url}"
+            return recs
+          end
+          http = Net::HTTP.new(url.host, url.port)
+          http.use_ssl = (url.scheme == 'https')
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          http.verify_depth = 5
+          request = Net::HTTP::Get.new(url.request_uri)
+          request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
           if @username
             request.basic_auth(@username, @password)
           end
           if @auth_token
             request.add_field("Authorization", @auth_token)
           end
-  				t1 = Time.now
-  				response = http.request(request)
-  				t2 = Time.now
-  				recs = parse_json(response.body, label, t2-t1)
-  				if limit
-  					recs[0,limit]
-  				else
-  					recs
-  				end
-  			}
-  		rescue Timeout::Error => e
-  			$stderr.puts "#{self.class.name} lookup timed out: #{label}"
-  		end
+          t1 = Time.now
+          0.upto(9) do
+            response = http.request(request)
+            body = response.body
+            if body == "Rate Limit Exceeded"
+              $stderr.puts "DEBUG: Rate Limit Exceeded. Retrying #{label}" if @debug
+            else
+              t2 = Time.now
+              recs  = parse_json(response.body, label, t2-t1)
+              break
+            end
+          end
+          if limit
+            recs[0,limit]
+          else
+            recs
+          end
+        }
+      rescue Timeout::Error
+        $stderr.puts "#{self.class.name} lookup timed out: #{label}"
+        recs
+      end
 
       private
 
       # parses the response of circl's JSON reply to generate an array of PDNSResult
-  		def parse_json(page,query,response_time=0)
-   			res = []
+      def parse_json(page,query,response_time=0)
+         res = []
         page.split(/\n/).each do |line|
           row = JSON.parse(line)
           firstseen = Time.at(row['time_first'].to_i)
           lastseen = Time.at(row['time_last'].to_i)
-  				res << PDNSResult.new(self.class.name,response_time,
+          res << PDNSResult.new(self.class.name,response_time,
             row['rrname'], row['rdata'], row['rrtype'], 0, 
-            firstseen, lastseen, row['count'])
+            firstseen, lastseen, row['count'], 'yellow')
         end
-  			res
-  		rescue Exception => e
-  			$stderr.puts "#{self.class.name} Exception: #{e}"
-  			raise e
-  		end
+        res
+      rescue Exception => e
+        $stderr.puts "#{self.class.name} Exception: #{e}"
+        raise e
+      end
 
-  	end
+    end
     CIRCL = PassiveDNS::Provider::Circl
   end
-end
+end

data/lib/passivedns/client/provider/cn360.rb

@@ -3,6 +3,7 @@ require 'net/https'
 require 'openssl'
 require 'json'
 require 'digest/md5'
+require 'pp'
 
 module PassiveDNS #:nodoc: don't document this
   # The Provider module contains all the Passive DNS provider client code
@@ -43,6 +44,7 @@ module PassiveDNS #:nodoc: don't document this
       #
       def initialize(options={})
         @debug = options[:debug] || false
+        @timeout = options[:timeout] || 20
         ["API", "API_ID", "API_KEY"].each do |opt|
           if not options[opt]
             raise "Field #{opt} is required.  See README.md"
@@ -60,52 +62,61 @@ module PassiveDNS #:nodoc: don't document this
         end
         limit ||= 10000
         path = "/api/#{table}/keyword/#{label}/count/#{limit}/"
-        url = @cp["API"]+path
-  			url = URI.parse url
-  			http = Net::HTTP.new(url.host, url.port)
-  			http.use_ssl = (url.scheme == 'https')
-  			http.verify_mode = OpenSSL::SSL::VERIFY_NONE # I hate doing this
-  			http.verify_depth = 5
-  			request = Net::HTTP::Get.new(url.path)
-  			request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
-        request.add_field('Accept', 'application/json')
-        request.add_field("X-BashTokid", @cp["API_ID"])
-        token = Digest::MD5.hexdigest(path+@cp["API_KEY"])
-  			$stderr.puts "DEBUG: cn360 url = #{url} token = #{token}" if @debug
-        request.add_field("X-BashToken", token)
-  			t1 = Time.now
-  			response = http.request(request)
-  			t2 = Time.now
-  			recs = parse_json(response.body, label, t2-t1)
-        if limit
-          recs[0,limit]
-        else
-          recs
-        end
+        Timeout::timeout(@timeout) {
+          url = @cp["API"]+path
+          url = URI.parse url
+          http = Net::HTTP.new(url.host, url.port)
+          http.use_ssl = (url.scheme == 'https')
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE # I hate doing this
+          http.verify_depth = 5
+          request = Net::HTTP::Get.new(url.path)
+          request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
+          request.add_field('Accept', 'application/json')
+          request.add_field("X-BashTokid", @cp["API_ID"])
+          token = Digest::MD5.hexdigest(path+@cp["API_KEY"])
+          $stderr.puts "DEBUG: cn360 url = #{url} token = #{token}" if @debug
+          request.add_field("X-BashToken", token)
+          t1 = Time.now
+          response = http.request(request)
+          t2 = Time.now
+          recs = parse_json(response.body, label, t2-t1)
+          if limit
+            recs[0,limit]
+          else
+            recs
+          end
+        }
+      rescue Timeout::Error
+        $stderr.puts "#{self.class.name} lookup timed out: #{label}"
+        recs
       end
       
       private
       
       # parses the response of 360.cn's JSON reply to generate an array of PDNSResult
       def parse_json(page,query,response_time=0)
-  			res = []
-  			data = JSON.parse(page)
+        res = []
+        data = JSON.parse(page)
+        if data.class == Hash and data['err']
+          raise "#{self.class.name} Error: #{data['err']}"
+        end
         data.each do |row|
-          time_first = (row["time_first"]) ? Time.at(row["time_first"].to_i) : nil
-          time_last = (row["time_last"]) ? Time.at(row["time_last"].to_i) : nil
+          time_first = (row["time_first"]) ? Time.at(row["time_first"]) : nil
+          time_last = (row["time_last"]) ? Time.at(row["time_last"]) : nil
           count = row["count"] || 0
           query = row["rrname"]
           answers = row["rdata"].gsub(/;$/,'').split(/;/)
           rrtype = row["rrtype"]
           answers.each do |answer|
-            res << PDNSResult.new(self.class.name, response_time, query, answer, rrtype, time_first, time_last, count)
+            res << PDNSResult.new(self.class.name, response_time, query, answer, rrtype, nil, time_first, time_last, count, TLPSecurityControl.new('yellow'))
           end
-  			end
-  			res
-  		rescue Exception => e
-  			$stderr.puts "#{self.class.name} Exception: #{e}"
-  			raise e
+        end
+        res
+      rescue Exception => e
+        $stderr.puts "#{self.class.name} Exception: #{e}"
+        puts e.backtrace
+        raise e
       end
     end
   end
-end
+end

data/lib/passivedns/client/provider/dnsdb.rb

@@ -8,7 +8,7 @@ module PassiveDNS #:nodoc: don't document this
   # The Provider module contains all the Passive DNS provider client code
   module Provider
     # Queries FarSight's passive DNS database
-  	class DNSDB < PassiveDB
+    class DNSDB < PassiveDB
       # Sets the modules self-reported name to "DNSDB"
       def self.name
         "DNSDB"
@@ -39,80 +39,81 @@ module PassiveDNS #:nodoc: don't document this
       #
       #   PassiveDNS::Provider::DNSDB.new(options)
       #
-  		def initialize(options={})
-  			@debug = options[:debug] || false
+      def initialize(options={})
+        @debug = options[:debug] || false
+        @timeout = options[:timeout] || 20
         @key = options["APIKEY"] || raise("APIKEY option required for #{self.class}")
         @base = options["URL"] || "https://api.dnsdb.info/lookup"
-  		end
+      end
 
       # Takes a label (either a domain or an IP address) and returns
       # an array of PassiveDNS::PDNSResult instances with the answers to the query
-  		def lookup(label, limit=nil)
-  			$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
-  			Timeout::timeout(240) {
-  				url = nil
-  				if label =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(\/\d{1,2})?$/
-  					label = label.gsub(/\//,',')
-  					url = "#{@base}/rdata/ip/#{label}"
-  				else
-  					url = "#{@base}/rrset/name/#{label}"
-  				end
-  				url = URI.parse url
-  				http = Net::HTTP.new(url.host, url.port)
-  				http.use_ssl = (url.scheme == 'https')
-  				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-  				http.verify_depth = 5
+      def lookup(label, limit=nil)
+        $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
+        Timeout::timeout(@timeout) {
+          url = nil
+          if label =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(\/\d{1,2})?$/
+            label = label.gsub(/\//,',')
+            url = "#{@base}/rdata/ip/#{label}"
+          else
+            url = "#{@base}/rrset/name/#{label}"
+          end
+          url = URI.parse url
+          http = Net::HTTP.new(url.host, url.port)
+          http.use_ssl = (url.scheme == 'https')
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          http.verify_depth = 5
           path = url.path
           if limit
             path << "?limit=#{limit}"
           end
-  				request = Net::HTTP::Get.new(path)
-  				request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
-  				request.add_field("X-API-Key", @key)
-  				request.add_field("Accept", "application/json")
-  				t1 = Time.now
-  				response = http.request(request)
-  				t2 = Time.now
-  				$stderr.puts response.body if @debug
-  				parse_json(response.body,t2-t1)
-  			}
-  		rescue Timeout::Error => e
-  			$stderr.puts "#{self.class.name} lookup timed out: #{label}"
-  		end
+          request = Net::HTTP::Get.new(path)
+          request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
+          request.add_field("X-API-Key", @key)
+          request.add_field("Accept", "application/json")
+          t1 = Time.now
+          response = http.request(request)
+          t2 = Time.now
+          $stderr.puts response.body if @debug
+          parse_json(response.body,t2-t1)
+        }
+      rescue Timeout::Error
+        $stderr.puts "#{self.class.name} lookup timed out: #{label}"
+      end
       
       private
     
       # parses the response of DNSDB's JSON reply to generate an array of PDNSResult
-  		def parse_json(page,response_time)
-  			res = []
-  			raise "Error: unable to parse request" if page =~ /Error: unable to parse request/
-  			rows = page.split(/\n/)
-  			rows.each do |row|
-  				record = JSON.parse(row)
+      def parse_json(page,response_time)
+        res = []
+        raise "Error: unable to parse request" if page =~ /Error: unable to parse request/
+        rows = page.split(/\n/)
+        rows.each do |row|
+          record = JSON.parse(row)
           answers = record['rdata']
-  				answers = [record['rdata']] if record['rdata'].class == String
+          answers = [record['rdata']] if record['rdata'].class == String
           query = record['rrname'].gsub!(/\.$/,'')
           rrtype = record['rrtype']
           firstseen = Time.at(record['time_first'].to_i)
           lastseen = Time.at(record['time_last'].to_i)
           count = record['count']
           
-  				answers.each do |answer|
+          answers.each do |answer|
             answer.gsub!(/\.$/,'')
-  					if record['time_first']
-  						res << PDNSResult.new(self.class.name,response_time,query,answer,rrtype,
-                0,firstseen,lastseen,count)
-  					else
-  						res << PDNSResult.new(self.class.name,response_time,query,answer,rrtype)
-  					end
-  				end
-  			end
-  			res
-  		rescue Exception => e
-  			$stderr.puts "#{self.class.name} Exception: #{e}"
-  			$stderr.puts page
-  			raise e
-  		end
-  	end    
+            if record['time_first']
+              res << PDNSResult.new(self.class.name,response_time,query,answer,rrtype,
+                0,firstseen,lastseen,count, 'yellow')
+            else
+              res << PDNSResult.new(self.class.name,response_time,query,answer,rrtype)
+            end
+          end
+        end
+        res
+      rescue Exception => e
+        $stderr.puts "#{self.class.name} Exception: #{e}"
+        $stderr.puts page
+        raise e
+      end
+    end    
   end
-end
+end