passivedns-client 2.0.6 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9fc3d708ad9c1dd55d73acf19b347589c8142292
-  data.tar.gz: f48765e8140f0f84dd26a8912728190458ab3dbb
+  metadata.gz: 9bc0b0ff155e1b9570c05b45f22368b0cfa79a83
+  data.tar.gz: 4b2d805d77a29ef052c0003a116c21224166254f
 SHA512:
-  metadata.gz: 6c6d15bd3b5e0c556b515512be6645b626139f9c0c2c42c8e48bb7347599edb2bcbb6ca0b011dd8c981a2fdd2d13f50e98f1d34935f81223a60039c237531052
-  data.tar.gz: d14cf8d3a6e50d8e1708d4fa32d11655efec83403ca9ba62fd10c3246dac1b7930d59ace1245a1cd5079270d22fbe7eba3124897b0b6bd95c8518f6346c13e12
+  metadata.gz: 21dd2aefcc5074d626047c4cf4e798c33b9dc063199667b29a68614463b464e01f91fe72b285ecf2a3a752f8ef1f9a4082c1da797f6ed405950becefa32e5c55
+  data.tar.gz: 39c7c7ae6f931634c58c588a9ce7113bdcd776becc587036e6a51261ba7d67008619b9cdf9da478674ba34cc46d255a965a9dc6dbaec83c9092e59daa95a2b49

data/.gitignore

@@ -8,6 +8,7 @@ InstalledFiles
 _yardoc
 coverage
 doc/
+html/
 lib/bundler/man
 pkg
 rdoc

data/README.md

@@ -8,6 +8,7 @@ This rubygem queries the following Passive DNS databases:
 * Mnemonic
 * PassiveDNS.cn (Qihoo 360 Technology Co.,Ltd)
 * PassiveTotal
+* RiskIQ
 * TCPIPUtils
 * VirusTotal
 
@@ -54,6 +55,9 @@ From version 2.0.0 on, all configuration keys for passive DNS providers are in o
 	[circl]
 	USERNAME = circl_user
 	PASSWORD = circl_pass
+	[riskiq]
+	API_TOKEN = 0123456789abcdef
+	API_PRIVATE_KEY = 01234567890abcdefghijklmnopqrstu
 
 CIRCL also can use and authorization token.  In that case, you should drop the USERNAME and PASSWORD options and change the section to something like the following:
 
@@ -67,6 +71,7 @@ CIRCL also can use and authorization token.  In that case, you should drop the U
 * DNSDB (Farsight Security) : https://api.dnsdb.info/
 * Mnemonic : mss .at. mnemonic.no
 * PassiveTotal : https://www.passivetotal.org
+* RiskIQ : https://github.com/RiskIQ/python_api/blob/master/LICENSE
 * TCPIPUtils : http://www.tcpiputils.com/premium-access
 * VirusTotal : https://www.virustotal.com
 
@@ -88,6 +93,7 @@ Or use the included tool...
 	  -dd use DNSDB
 	  -dm use Mnemonic
 	  -dp use PassiveTotal
+	  -dr use RiskIQ
 	  -dt use TCPIPUtils
 	  -dv use VirusTotal
 	  -dvt uses VirusTotal and TCPIPUtils (for example)

data/Rakefile

@@ -2,6 +2,7 @@
 require "bundler/gem_tasks"
 
 require 'rake/testtask'
+require 'rdoc/task'
 
 Rake::TestTask.new do |t|
 	t.libs << 'lib'
@@ -9,4 +10,11 @@ Rake::TestTask.new do |t|
 	t.verbose = true
 end
 
+RDoc::Task.new do |rd|
+  rd.main = "README.doc"
+  rd.rdoc_files.include("README.md", "lib/**/*.rb")
+  rd.options << "--all"
+  rd.options << "--verbose"
+end
+
 task :default => :test

data/lib/passivedns/client.rb

@@ -8,27 +8,32 @@ require 'passivedns/client/passivedb'
 
 # load all the providers
 $passivedns_providers = Array.new
-provider_path = File.dirname(__FILE__)+"/client/providers/*.rb"
+provider_path = File.dirname(__FILE__)+"/client/provider/*.rb"
 Dir.glob(provider_path).each do |provider|
   name = File.basename(provider, '.rb')
-  require "passivedns/client/providers/#{name}.rb"
+  require "passivedns/client/provider/#{name}.rb"
   $passivedns_providers << name
 end
 
 require 'configparser'
 
-module PassiveDNS
-
+module PassiveDNS # :nodoc:
+  # struct to contain the results from a PassiveDNS lookup
 	class PDNSResult < Struct.new(:source, :response_time, :query, :answer, :rrtype, :ttl, :firstseen, :lastseen, :count); end
 
+  # coodinates the lookups accross all configured PassiveDNS providers
 	class Client
+    
+    # instantiate and configure all specified PassiveDNS providers
+    # pdns        array of passivedns provider names, e.g., ["dnsdb","virustotal"]
+    # configfile  filename of the passivedns-client configuration (this should probably be abstracted)
 		def initialize(pdns=$passivedns_providers, configfile="#{ENV['HOME']}/.passivedns-client")
       cp = ConfigParser.new(configfile)
       # this creates a map of all the PassiveDNS provider names and their classes
       class_map = {}
-      PassiveDNS.constants.each do |const|
-        if PassiveDNS.const_get(const).is_a?(Class) and PassiveDNS.const_get(const).superclass == PassiveDNS::PassiveDB
-          class_map[PassiveDNS.const_get(const).config_section_name] = PassiveDNS.const_get(const)
+      PassiveDNS::Provider.constants.each do |const|
+        if PassiveDNS::Provider.const_get(const).is_a?(Class) and PassiveDNS::Provider.const_get(const).superclass == PassiveDNS::PassiveDB
+          class_map[PassiveDNS::Provider.const_get(const).config_section_name] = PassiveDNS::Provider.const_get(const)
         end
       end
       
@@ -43,12 +48,14 @@ module PassiveDNS
 
 		end #initialize
 		
+    # set the debug flag
 		def debug=(d)
 			@pdnsdbs.each do |pdnsdb|
 				pdnsdb.debug = d
 			end
 		end
 		
+    # perform the query lookup accross all configured PassiveDNS providers
 		def query(item, limit=nil)
 			threads = []
 			@pdnsdbs.each do |pdnsdb|

data/lib/passivedns/client/cli.rb

@@ -4,18 +4,39 @@ require 'getoptlong'
 require 'yaml'
 require 'pp'
 
-module PassiveDNS
+module PassiveDNS # :nodoc:
+  # Handles all the command-line parsing, state tracking, and dispatching queries to the PassiveDNS::Client instance
+  # CLInterface is aliased by CLI
 	class CLInterface
+    #  generates a mapping between the option letter for each PassiveDNS provider and the class
     def self.get_letter_map
       letter_map = {}
-      PassiveDNS.constants.each do |const|
-        if PassiveDNS.const_get(const).is_a?(Class) and PassiveDNS.const_get(const).superclass == PassiveDNS::PassiveDB
-          letter_map[PassiveDNS.const_get(const).option_letter] = [PassiveDNS.const_get(const).name, PassiveDNS.const_get(const).config_section_name]
+      mod = PassiveDNS::Provider
+      mod.constants.each do |const|
+        if mod.const_get(const).is_a?(Class) and mod.const_get(const).superclass == PassiveDNS::PassiveDB
+          letter = mod.const_get(const).option_letter
+          name = mod.const_get(const).name
+          config_section_name = mod.const_get(const).config_section_name
+          letter_map[letter] = [name, config_section_name]
         end
       end
       letter_map      
     end
     
+    # parses the command line and yields an options hash
+    # === Default Options
+    #      options = {
+    #        :pdnsdbs => [],     # passive dns providers to query
+    #        :format => "text",  # output format
+    #        :sep => "\t",       # field separator for text format
+    #        :recursedepth => 1, # recursion depth
+    #        :wait => 0,         # wait period between recursions
+    #        :res => nil,        # unused.  I don't remember why this is here.
+    #        :debug => false,    # debug flag
+    #        :sqlitedb => nil,   # filename for maintaining state in an sqlite3 db
+    #        :limit => nil,      # number of results per provider per recursion
+    #        :help => false      # display the usage text
+    #      }
     def self.parse_command_line(args)
       origARGV = ARGV.dup
       ARGV.replace(args)
@@ -132,6 +153,8 @@ module PassiveDNS
       [options, args]
     end
     
+    # returns a string containing the usage information
+    # takes in a hash of letter to passive dns providers
     def self.usage(letter_map)
       databases = letter_map.keys.sort.join("")
       help_text = ""
@@ -167,6 +190,7 @@ module PassiveDNS
     	help_text
     end
     
+    # performs a stateful, recursive (if desired) passive DNS lookup against all specified providers
     def self.pdnslookup(state, pdnsclient, options)
       recursedepth = options[:recursedepth]
       wait = options[:wait]
@@ -195,6 +219,7 @@ module PassiveDNS
     	state
     end
     
+    # returns a string transforming all the PassiveDNS::PDNSResult stored in the state object into text/xml/json/etc.
     def self.results_to_s(state,options)
       format = options[:format]
       sep = options[:sep]
@@ -216,6 +241,7 @@ module PassiveDNS
     	end
     end
     
+    # create a state instance
     def self.create_state(sqlitedb=nil)
       state = nil
       if sqlitedb
@@ -225,6 +251,7 @@ module PassiveDNS
       end
     end
     
+    # main method, takes command-line arguments and performs the desired queries and outputs
     def self.run(args)
       options, items = parse_command_line(args)
       if options[:help]
@@ -240,8 +267,8 @@ module PassiveDNS
       pdnsclient = PassiveDNS::Client.new(options[:pdnsdbs])
       pdnsclient.debug = options[:debug]
       
-      if ARGV.length > 0
-      	ARGV.each do |arg|
+      if items.length > 0
+      	items.each do |arg|
       		state.add_query(arg,'pending',0)
       	end
       else
@@ -253,6 +280,7 @@ module PassiveDNS
       results_to_s(state,options)
     end
   end
+  # Alias for the CLInterface class
   CLI = PassiveDNS::CLInterface
 end
 

data/lib/passivedns/client/passivedb.rb

@@ -1,17 +1,22 @@
-module PassiveDNS
+module PassiveDNS #:nodoc: don't document this
+  # abstract class that all PassiveDNS::Provider should subclass
   class PassiveDB
+    # raises an exception that this should be implemented by the subclass
     def self.name
       raise "You should implement your own version of .name"
     end
     
+    # raises an exception that this should be implemented by the subclass
     def self.config_section_name
       name
     end
     
+    # raises an exception that this should be implemented by the subclass
     def self.option_letter
       raise "You should pick a unique letter to serve as your database option letter for the command line option -d"
     end
     
+    # raises an exception that this should be implemented by the subclass
     def lookup(label, limit=nil)
       raise "You must implement the lookup function"
     end

data/lib/passivedns/client/provider/bfk.rb

@@ -0,0 +1,102 @@
+require 'open-uri'
+
+module PassiveDNS #:nodoc: don't document this
+  # The Provider module contains all the Passive DNS provider client code
+  module Provider
+
+    # Queries BFK.de's passive DNS database
+  	class BFK < PassiveDB
+      # Sets the modules self-reported name to "BFK.de"
+      def self.name
+        "BFK.de"
+      end
+      # Sets the configuration section name to "bfk"
+      def self.config_section_name
+        "bfk"
+      end
+      # Sets the command line database argument to "b"
+      def self.option_letter
+        "b"
+      end
+    
+      # :debug enables verbose logging to standard output
+      attr_accessor :debug
+      # === Options
+      # * :debug     Sets the debug flag for the module
+      # * "URL"      Alternate url for testing.  Defaults to "http://www.bfk.de/bfk_dnslogger.html?query="
+      #
+      # === Example Instantiation
+      #
+      #   options = {
+      #     :debug => true,
+      #     "URL" => "http://www.bfk.de/bfk_dnslogger.html?query="
+      #   }
+      #
+      #   PassiveDNS::Provider::BFK.new(options)
+      #
+      
+  		def initialize(options={})
+  			@debug = options[:debug] || false
+        @base = options["URL"] || "http://www.bfk.de/bfk_dnslogger.html?query="
+  		end
+    
+      # Takes a label (either a domain or an IP address) and returns
+      # an array of PassiveDNS::PDNSResult instances with the answers to the query
+   		def lookup(label, limit=nil)	
+  			$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
+  			Timeout::timeout(240) {
+  				t1 = Time.now
+  				open(
+  					@base+label,
+  					"User-Agent" => "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}"
+  				) do |f|
+  					t2 = Time.now
+  					recs = parse(f.read,t2-t1)
+            if limit
+              recs[0,limit]
+            else
+              recs
+            end
+  				end
+  			}
+  		rescue Timeout::Error => e
+  			$stderr.puts "#{self.class.name} lookup timed out: #{label}"      
+  		end
+    
+      private
+    
+      # parses the webpage returned by BFK to generate an array of PDNSResult
+  		def parse(page,response_time)
+  			line = page.unpack('C*').pack('U*').split(/<table/).grep(/ id=\"logger\"/)
+  			return [] unless line.length > 0
+  			line = line[0].gsub(/[\t\n]/,'').gsub(/<\/table.*/,'')
+  			rows = line.split(/<tr.*?>/)
+  			res = []
+  			rows.collect do |row|
+  				r = row.split(/<td>/).map{|x| x.gsub(/<.*?>/,'').gsub(/\&.*?;/,'')}[1,1000]
+  				if r and r[0] =~ /\w/
+  					# BFK includes the MX weight in the answer response. First, find the MX records, then dump the weight to present a consistent record name to the collecting array. Otherwise the other repositories will present the same answer and your results will become cluttered with duplicates.
+  					if r[1] == "MX" then
+  						# MX lines look like "5 mx.domain.tld", so split on the space and assign r[2] (:answer) to the latter part.
+  						#s = r[2].split(/\w/).map{|x| x}[1,1000]
+  						#	r[2] = s[1]
+  						r[2] =~ /[0-9]+?\s(.+)/
+  						s=$1
+  						puts "DEBUG: == BFK: MX Parsing Strip: Answer: " + r[2] + " : mod: " + s if @debug
+  						r[2] = s
+						
+  							######### FIX BLANKS FOR MX
+						
+  					end
+  					res << PDNSResult.new(self.class.name,response_time,r[0],r[2],r[1])
+  				end
+  			end
+  			res
+  		rescue Exception => e
+  			$stderr.puts "#{self.class.name} Exception: #{e}"
+  			raise e
+  		end
+
+  	end
+  end
+end

data/lib/passivedns/client/provider/circl.rb

@@ -0,0 +1,111 @@
+# DESCRIPTION: Module to query PassiveTotal's passive DNS repository
+
+require 'net/http'
+require 'net/https'
+require 'openssl'
+
+module PassiveDNS #:nodoc: don't document this
+  # The Provider module contains all the Passive DNS provider client code
+  module Provider
+    # Queries CIRCL.LU's passive DNS database
+    # Circl is aliased by CIRCL
+  	class Circl < PassiveDB
+      # Sets the modules self-reported name to "CIRCL"
+      def self.name
+        "CIRCL"
+      end
+      # Sets the configuration section name to "circl"
+      def self.config_section_name
+        "circl"
+      end
+      # Sets the command line database argument to "c"
+      def self.option_letter
+        "c"
+      end
+    
+      # :debug enables verbose logging to standard output
+      attr_accessor :debug
+      # === Options
+      # * :debug       Sets the debug flag for the module
+      # * "USERNAME"   User name associated with your CIRCL account
+      # * "PASSWORD"   Password associated with your CIRCL account
+      # * "AUTH_TOKEN" Authorization token associated with your CIRCL account
+      # * "URL"      Alternate url for testing.  Defaults to "https://www.circl.lu/pdns/query"
+      
+      # You should either have a username+password or an authorization token to use this service
+      #
+      # === Example Instantiation
+      #
+      #   options = {
+      #     :debug => true,
+      #     "USERNAME" => "circl_user",
+      #     "PASSWORD" => "circl_pass",
+      #     "URL" => "https://www.circl.lu/pdns/query"
+      #   }
+      #
+      #   PassiveDNS::Provider::CIRCL.new(options)
+      #
+  		def initialize(options={})
+        @debug = options[:debug] || false
+        @username = options["USERNAME"]
+        @password = options["PASSWORD"]
+        @auth_token = options["AUTH_TOKEN"]
+        @url = options["URL"] || "https://www.circl.lu/pdns/query"
+  		end
+
+      # Takes a label (either a domain or an IP address) and returns
+      # an array of PassiveDNS::PDNSResult instances with the answers to the query
+  		def lookup(label, limit=nil)
+  			$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
+  			Timeout::timeout(240) {
+  				url = @url+"/"+label
+  				$stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
+  				url = URI.parse url
+  				http = Net::HTTP.new(url.host, url.port)
+  				http.use_ssl = (url.scheme == 'https')
+  				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+  				http.verify_depth = 5
+  				request = Net::HTTP::Get.new(url.request_uri)
+  				request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
+          if @username
+            request.basic_auth(@username, @password)
+          end
+          if @auth_token
+            request.add_field("Authorization", @auth_token)
+          end
+  				t1 = Time.now
+  				response = http.request(request)
+  				t2 = Time.now
+  				recs = parse_json(response.body, label, t2-t1)
+  				if limit
+  					recs[0,limit]
+  				else
+  					recs
+  				end
+  			}
+  		rescue Timeout::Error => e
+  			$stderr.puts "#{self.class.name} lookup timed out: #{label}"
+  		end
+
+      private
+
+      # parses the response of circl's JSON reply to generate an array of PDNSResult
+  		def parse_json(page,query,response_time=0)
+   			res = []
+  			# need to remove the json_class tag or the parser will crap itself trying to find a class to align it to
+        page.split(/\n/).each do |line|
+          row = JSON.parse(line)
+  				res << PDNSResult.new(self.class.name,response_time,
+            row['rrname'], row['rdata'], row['rrtype'], 0, 
+            row['time_first'], row['time_last'], row['count'])
+        end
+  			res
+  		rescue Exception => e
+  			$stderr.puts "#{self.class.name} Exception: #{e}"
+  			raise e
+  		end
+
+  	end
+    CIRCL = PassiveDNS::Provider::Circl
+  end
+end