passivedns-client 2.0.6 → 2.1.0

data/lib/passivedns/client/provider/cn360.rb

@@ -0,0 +1,108 @@
+require 'net/http'
+require 'net/https'
+require 'openssl'
+require 'json'
+require 'digest/md5'
+
+module PassiveDNS #:nodoc: don't document this
+  # The Provider module contains all the Passive DNS provider client code
+  module Provider
+    # Queries 360.cn's passive DNS database
+    class CN360 < PassiveDB
+      # Sets the modules self-reported name to "360.cn"
+      def self.name
+        "360.cn"
+      end
+      # Sets the configuration section name to "cn360"
+      def self.config_section_name
+        "cn360"
+      end
+      # Sets the command line database argument to "3"
+      def self.option_letter
+        "3"
+      end
+    
+      # :debug enables verbose logging to standard output
+      attr_accessor :debug
+      # === Options
+      # * :debug       Sets the debug flag for the module
+      # * "API"        REQUIRED: http://some.web.address.for.their.api
+      # * "API_ID"     REQUIRED: a username that is given when you register
+      # * "API_KEY"    REQUIRED: a long and random password of sorts that is used along with the page request to generate a per page API key
+      #
+      # === Example Instantiation
+      #
+      #   options = {
+      #     :debug => true,
+      #     "API" => "http://some.web.address.for.their.api",
+      #     "API_ID" => "360user",
+      #     "API_KEY" => "360apikey"
+      #   }
+      #
+      #   PassiveDNS::Provider::CN360.new(options)
+      #
+      def initialize(options={})
+        @debug = options[:debug] || false
+        ["API", "API_ID", "API_KEY"].each do |opt|
+          if not options[opt]
+            raise "Field #{opt} is required.  See README.md"
+          end
+        end
+        @cp = options
+      end
+    
+      # Takes a label (either a domain or an IP address) and returns
+      # an array of PassiveDNS::PDNSResult instances with the answers to the query
+      def lookup(label, limit=10000)
+        table = "rrset"
+        if label =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/ or label =~ /^[0-9a-fA-F]+:[0-9a-fA-F:]+[0-9a-fA-F]$/
+          table = "rdata"
+        end
+        limit ||= 10000
+        path = "/api/#{table}/keyword/#{label}/count/#{limit}/"
+        url = @cp["API"]+path
+  			url = URI.parse url
+  			http = Net::HTTP.new(url.host, url.port)
+  			http.use_ssl = (url.scheme == 'https')
+  			http.verify_mode = OpenSSL::SSL::VERIFY_NONE # I hate doing this
+  			http.verify_depth = 5
+  			request = Net::HTTP::Get.new(url.path)
+  			request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
+        request.add_field('Accept', 'application/json')
+        request.add_field("X-BashTokid", @cp["API_ID"])
+        token = Digest::MD5.hexdigest(path+@cp["API_KEY"])
+  			$stderr.puts "DEBUG: cn360 url = #{url} token = #{token}" if @debug
+        request.add_field("X-BashToken", token)
+  			t1 = Time.now
+  			response = http.request(request)
+  			t2 = Time.now
+  			recs = parse_json(response.body, label, t2-t1)
+        if limit
+          recs[0,limit]
+        else
+          recs
+        end
+      
+        private
+      
+        # parses the response of 360.cn's JSON reply to generate an array of PDNSResult
+        def parse_json(page,query,response_time=0)
+    			res = []
+    			# need to remove the json_class tag or the parser will crap itself trying to find a class to align it to
+    			data = JSON.parse(page)
+          data.each do |row|
+            time_first = (row["time_first"]) ? Time.at(row["time_first"].to_i) : nil
+            time_last = (row["time_last"]) ? Time.at(row["time_last"].to_i) : nil
+            count = row["count"] || 0
+            res << PDNSResult.new(self.class.name, response_time, row["rrname"], row["rdata"], row["rrtype"], time_first, time_last, count)
+    			end
+    			res
+    		rescue Exception => e
+    			$stderr.puts "#{self.class.name} Exception: #{e}"
+    			raise e
+        end
+    
+      end      
+    end
+  end
+end

data/lib/passivedns/client/provider/dnsdb.rb

@@ -0,0 +1,110 @@
+# DESCRIPTION: this is a module for pdns.rb, primarily used by pdnstool.rb, to query the Farsight Security passive DNS database
+# details on the API are at https://api.dnsdb.info/
+# to request an API key, please email dnsdb-api at farsightsecurity dot com.
+require 'net/http'
+require 'net/https'
+
+module PassiveDNS #:nodoc: don't document this
+  # The Provider module contains all the Passive DNS provider client code
+  module Provider
+    # Queries FarSight's passive DNS database
+  	class DNSDB < PassiveDB
+      # Sets the modules self-reported name to "DNSDB"
+      def self.name
+        "DNSDB"
+      end
+      # Sets the configuration section name to "dnsdb"
+      def self.config_section_name
+        "dnsdb"
+      end
+      # Sets the command line database argument to "d"
+      def self.option_letter
+        "d"
+      end
+    
+      # :debug enables verbose logging to standard output
+      attr_accessor :debug
+      # === Options
+      # * :debug       Sets the debug flag for the module
+      # * "APIKEY"     REQUIRED: The API key associated with DNSDB
+      # * "URL"      Alternate url for testing.  Defaults to "https://api.dnsdb.info/lookup"
+      #
+      # === Example Instantiation
+      #
+      #   options = {
+      #     :debug => true,
+      #     "APIKEY" => "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+      #     "URL" => "https://api.dnsdb.info/lookup"
+      #   }
+      #
+      #   PassiveDNS::Provider::DNSDB.new(options)
+      #
+  		def initialize(options={})
+  			@debug = options[:debug] || false
+        @key = options["APIKEY"] || raise("APIKEY option required for #{self.class}")
+        @base = options["URL"] || "https://api.dnsdb.info/lookup"
+  		end
+
+      # Takes a label (either a domain or an IP address) and returns
+      # an array of PassiveDNS::PDNSResult instances with the answers to the query
+  		def lookup(label, limit=nil)
+  			$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
+  			Timeout::timeout(240) {
+  				url = nil
+  				if label =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(\/\d{1,2})?$/
+  					label = label.gsub(/\//,',')
+  					url = "#{@base}/rdata/ip/#{label}"
+  				else
+  					url = "#{@base}/rrset/name/#{label}"
+  				end
+  				url = URI.parse url
+  				http = Net::HTTP.new(url.host, url.port)
+  				http.use_ssl = (url.scheme == 'https')
+  				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+  				http.verify_depth = 5
+          path = url.path
+          if limit
+            path << "?limit=#{limit}"
+          end
+  				request = Net::HTTP::Get.new(path)
+  				request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
+  				request.add_field("X-API-Key", @key)
+  				request.add_field("Accept", "application/json")
+  				t1 = Time.now
+  				response = http.request(request)
+  				t2 = Time.now
+  				$stderr.puts response.body if @debug
+  				parse_json(response.body,t2-t1)
+  			}
+  		rescue Timeout::Error => e
+  			$stderr.puts "#{self.class.name} lookup timed out: #{label}"
+  		end
+      
+      private
+    
+      # parses the response of DNSDB's JSON reply to generate an array of PDNSResult
+  		def parse_json(page,response_time)
+  			res = []
+  			raise "Error: unable to parse request" if page =~ /Error: unable to parse request/
+  			# need to remove the json_class tag or the parser will crap itself trying to find a class to align it to
+  			rows = page.split(/\n/)
+  			rows.each do |row|
+  				record = JSON.parse(row)
+  				record['rdata'] = [record['rdata']] if record['rdata'].class == String
+  				record['rdata'].each do |rdata|
+  					if record['time_first']
+  						res << PDNSResult.new(self.class.name,response_time,record['rrname'],rdata,record['rrtype'],0,Time.at(record['time_first'].to_i).utc.strftime("%Y-%m-%dT%H:%M:%SZ"),Time.at(record['time_last'].to_i).utc.strftime("%Y-%m-%dT%H:%M:%SZ"),record['count'])
+  					else
+  						res << PDNSResult.new(self.class.name,response_time,record['rrname'],rdata,record['rrtype'])
+  					end
+  				end
+  			end
+  			res
+  		rescue Exception => e
+  			$stderr.puts "#{self.class.name} Exception: #{e}"
+  			$stderr.puts page
+  			raise e
+  		end
+  	end    
+  end
+end

data/lib/passivedns/client/provider/mnemonic.rb

@@ -0,0 +1,98 @@
+# DESCRIPTION: Module to query Mnemonic's passive DNS repository
+# CONTRIBUTOR: Drew Hunt (pinowudi@yahoo.com)
+require 'net/http'
+require 'net/https'
+require 'openssl'
+
+module PassiveDNS #:nodoc: don't document this
+  # The Provider module contains all the Passive DNS provider client code
+  module Provider
+    # Queries Mnemonic's passive DNS database
+  	class Mnemonic < PassiveDB
+      # Sets the modules self-reported name to "Mnemonic"
+      def self.name
+        "Mnemonic"
+      end
+      # Sets the configuration section name to "mnemonic"
+      def self.config_section_name
+        "mnemonic"
+      end
+      # Sets the command line database argument to "m"
+      def self.option_letter
+        "m"
+      end
+    
+      # :debug enables verbose logging to standard output
+      attr_accessor :debug
+      # === Options
+      # * :debug       Sets the debug flag for the module
+      # * "APIKEY"     REQUIRED: The API key associated with Mnemonic
+      # * "URL"      Alternate url for testing.  Defaults to "https://passivedns.mnemonic.no/api1/?apikey="
+      #
+      # === Example Instantiation
+      #
+      #   options = {
+      #     :debug => true,
+      #     "APIKEY" => "01234567890abcdef01234567890abcdef012345",
+      #     "URL" => "https://passivedns.mnemonic.no/api1/?apikey="
+      #   }
+      #
+      #   PassiveDNS::Provider::Mnemonic.new(options)
+      #
+  		def initialize(options={})
+        @debug = options[:debug] || false
+        @apikey = options["APIKEY"] || raise("#{self.class.name} requires an APIKEY")
+        @url = options["URL"] || "https://passivedns.mnemonic.no/api1/?apikey="
+  		end
+
+      # Takes a label (either a domain or an IP address) and returns
+      # an array of PassiveDNS::PDNSResult instances with the answers to the query
+  		def lookup(label, limit=nil)
+  			$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
+  			Timeout::timeout(240) {
+  				url = "#{@url}#{@apikey}&query=#{label}&method=exact"
+  				$stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
+  				url = URI.parse url
+  				http = Net::HTTP.new(url.host, url.port)
+  				http.use_ssl = (url.scheme == 'https')
+  				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+  				http.verify_depth = 5
+  				request = Net::HTTP::Get.new(url.path+"?"+url.query)
+  				request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
+  				t1 = Time.now
+  				response = http.request(request)
+  				t2 = Time.now
+  				recs = parse_json(response.body, label, t2-t1)
+  				if limit
+  					recs[0,limit]
+  				else
+  					recs
+  				end
+  			}
+  		rescue Timeout::Error => e
+  			$stderr.puts "#{self.class.name} lookup timed out: #{label}"
+  		end
+    
+      private
+    
+      # parses the response of mnemonic's JSON reply to generate an array of PDNSResult
+  		def parse_json(page,query,response_time=0)
+  			res = []
+  			# need to remove the json_class tag or the parser will crap itself trying to find a class to align it to
+  			data = JSON.parse(page)
+  			if data['result']
+  				data['result'].each do |row|
+  					if row['query']
+  						res << PDNSResult.new(self.class.name,response_time,row['query'],row['answer'],row['type'].upcase,row['ttl'],row['first'],row['last'])
+  					end
+  				end
+  			end
+  			res
+  		rescue Exception => e
+  			$stderr.puts "#{self.class.name} Exception: #{e}"
+  			raise e
+  		end
+
+  	end
+  end
+end

data/lib/passivedns/client/provider/passivetotal.rb

@@ -0,0 +1,103 @@
+# DESCRIPTION: Module to query PassiveTotal's passive DNS repository
+
+require 'net/http'
+require 'net/https'
+require 'openssl'
+
+module PassiveDNS #:nodoc: don't document this
+  # The Provider module contains all the Passive DNS provider client code
+  module Provider
+    # Queries PassiveTotal's passive DNS database
+  	class PassiveTotal < PassiveDB
+      # Sets the modules self-reported name to "PassiveTotal"
+      def self.name
+        "PassiveTotal"
+      end
+      # Sets the configuration section name to "passivetotal"
+      def self.config_section_name
+        "passivetotal"
+      end
+      # Sets the command line database argument to "p"
+      def self.option_letter
+        "p"
+      end
+    
+      # :debug enables verbose logging to standard output
+      attr_accessor :debug
+      # === Options
+      # * :debug       Sets the debug flag for the module
+      # * "APIKEY"     REQUIRED: The API key associated with PassiveTotal
+      # * "URL"      Alternate url for testing.  Defaults to "https://www.passivetotal.org/api/passive"
+      #
+      # === Example Instantiation
+      #
+      #   options = {
+      #     :debug => true,
+      #     "APIKEY" => "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+      #     "URL" => "https://www.passivetotal.org/api/passive"
+      #   }
+      #
+      #   PassiveDNS::Provider::PassiveTotal.new(options)
+      #
+  		def initialize(options={})
+        @debug = options[:debug] || false
+        @apikey = options["APIKEY"] || raise("#{self.class.name} requires an APIKEY")
+        @url = options["URL"] || "https://www.passivetotal.org/api/passive"
+  		end
+
+      # Takes a label (either a domain or an IP address) and returns
+      # an array of PassiveDNS::PDNSResult instances with the answers to the query
+  		def lookup(label, limit=nil)
+  			$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
+  			Timeout::timeout(240) {
+  				url = @url
+  				$stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
+  				url = URI.parse url
+  				http = Net::HTTP.new(url.host, url.port)
+  				http.use_ssl = (url.scheme == 'https')
+  				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+  				http.verify_depth = 5
+  				request = Net::HTTP::Post.new(url.request_uri)
+  				request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
+          request.set_form_data({"apikey" => @apikey, "value" => label})
+  				t1 = Time.now
+  				response = http.request(request)
+  				t2 = Time.now
+  				recs = parse_json(response.body, label, t2-t1)
+  				if limit
+  					recs[0,limit]
+  				else
+  					recs
+  				end
+  			}
+  		rescue Timeout::Error => e
+  			$stderr.puts "#{self.class.name} lookup timed out: #{label}"
+  		end
+    
+      private
+    
+      # parses the response of passivetotals's JSON reply to generate an array of PDNSResult
+  		def parse_json(page,query,response_time=0)
+   			res = []
+  			# need to remove the json_class tag or the parser will crap itself trying to find a class to align it to
+  			data = JSON.parse(page)
+  			if data['results']
+          query = data['results']['value']
+  				data['results']['resolutions'].each do |row|
+            first_seen = row['firstSeen']
+            last_seen = row['lastSeen']
+            value = row['value']
+            source = row['source'].join(",")
+  					res << PDNSResult.new(self.class.name+"/"+source,response_time,
+              query, value, "A", 0, first_seen, last_seen)
+  				end
+  			end
+  			res
+  		rescue Exception => e
+  			$stderr.puts "#{self.class.name} Exception: #{e}"
+  			raise e
+  		end
+
+  	end
+  end
+end

data/lib/passivedns/client/provider/riskiq.rb

@@ -0,0 +1,130 @@
+# DESCRIPTION: Module to query PassiveTotal's passive DNS repository
+
+require 'net/http'
+require 'net/https'
+require 'openssl'
+#require 'pp'
+
+module PassiveDNS #:nodoc: don't document this
+  # The Provider module contains all the Passive DNS provider client code
+  module Provider
+    # Queries RiskIQ's passive DNS database
+  	class RiskIQ < PassiveDB
+      # Sets the modules self-reported name to "RiskIQ"
+      def self.name
+        "RiskIQ"
+      end
+      # Sets the configuration section name to "riskiq"
+      def self.config_section_name
+        "riskiq"
+      end
+      # Sets the command line database argument to "r"
+      def self.option_letter
+        "r"
+      end
+    
+      # :debug enables verbose logging to standard output
+      attr_accessor :debug
+      # === Options
+      # * :debug            Sets the debug flag for the module
+      # * "API_TOKEN"       REQUIRED: User name associated with your RiskIQ account
+      # * "API_PRIVATE_KEY" REQUIRED: Password associated with your RiskIQ account
+      # * "API_SERVER"      Alternate server for testing.  Defaults to "ws.riskiq.net"
+      # * "API_VERSION"     Alternate version of the API to test.  Defaults to "V1"
+      #
+      # === Example Instantiation
+      #
+      #   options = {
+      #     :debug => true,
+      #     "API_TOKEN" => "riskiq_token",
+      #     "API_PRIVATE_KEY" => "riskiq_private_key",
+      #     "API_SERVER" => "ws.riskiq.net",
+      #     "API_VERSION" => "v1"
+      #   }
+      #
+      #   PassiveDNS::Provider::RiskIQ.new(options)
+      #
+  		def initialize(options={})
+        @debug = options[:debug] || false
+        @token = options["API_TOKEN"] || raise("#{self.class.name} requires an API_TOKEN")
+        @privkey = options["API_PRIVATE_KEY"] || raise("#{self.class.name} requires an API_PRIVATE_KEY")
+        @server = options["API_SERVER"] || "ws.riskiq.net"
+        @version = options["API_VERSION"] || "v1"
+        @url = "https://#{@server}/#{@version}"
+  		end
+
+      # Takes a label (either a domain or an IP address) and returns
+      # an array of PassiveDNS::PDNSResult instances with the answers to the query
+  		def lookup(label, limit=nil)
+  			$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
+  			Timeout::timeout(240) {
+          url = nil
+          params = {"rrType" => "", "maxResults" => limit || 1000}
+        
+          if label =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/
+            url = @url+"/dns/data"
+            params["ip"] = label 
+          else
+            url = @url+"/dns/name"
+            params["name"] = label
+          end
+          url << "?"
+          params.each do |k,v|
+            url << "#{k}=#{v}&"
+          end
+          url.gsub!(/\&$/,"")
+        
+  				$stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
+  				url = URI.parse url
+  				http = Net::HTTP.new(url.host, url.port)
+  				http.use_ssl = (url.scheme == 'https')
+  				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+  				http.verify_depth = 5
+  				request = Net::HTTP::Get.new(url.request_uri)
+  				request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
+          request.add_field('Accept', 'Application/JSON')
+          request.add_field('Content-Type', 'Application/JSON')
+          request.basic_auth(@token, @privkey)
+  				t1 = Time.now
+  				response = http.request(request)
+  				t2 = Time.now
+  				recs = parse_json(response.body, label, t2-t1)
+  				if limit
+  					recs[0,limit]
+  				else
+  					recs
+  				end
+  			}
+  		rescue Timeout::Error => e
+  			$stderr.puts "#{self.class.name} lookup timed out: #{label}"
+  		end
+    
+      private
+    
+      # parses the response of riskiq's JSON reply to generate an array of PDNSResult
+  		def parse_json(page,query,response_time=0)
+        #pp page
+   			res = []
+  			# need to remove the json_class tag or the parser will crap itself trying to find a class to align it to
+  			data = JSON.parse(page)
+  			if data['records']
+          data['records'].each do |record|
+            name = record['name']
+            type = record['rrtype']
+            last_seen = record['lastSeen']
+            first_seen = record['firstSeen']
+            record['data'].each do |datum|
+    					res << PDNSResult.new(self.class.name,response_time,
+                name, datum, type, 0, first_seen, last_seen)
+            end
+          end
+        end
+  			res
+  		rescue Exception => e
+  			$stderr.puts "#{self.class.name} Exception: #{e}"
+  			raise e
+  		end
+
+  	end
+  end
+end