passivedns-client 2.0.6 → 2.1.0

data/lib/passivedns/client/provider/tcpiputils.rb

@@ -0,0 +1,118 @@
+require 'net/http'
+require 'net/https'
+require 'openssl'
+require 'json'
+
+# Please read http://www.tcpiputils.com/terms-of-service under automated requests
+
+module PassiveDNS #:nodoc: don't document this
+  # The Provider module contains all the Passive DNS provider client code
+  module Provider
+    # Queries TCPIPUtils's passive DNS database
+    class TCPIPUtils < PassiveDB
+      # Sets the modules self-reported name to "TCPIPUtils"
+      def self.name
+        "TCPIPUtils"
+      end
+      # Sets the configuration section name to "tcpiputils"
+      def self.config_section_name
+        "tcpiputils"
+      end
+      # Sets the command line database argument to "t"
+      def self.option_letter
+        "t"
+      end
+
+      # :debug enables verbose logging to standard output
+      attr_accessor :debug
+      # === Options
+      # * :debug       Sets the debug flag for the module
+      # * "APIKEY"     REQUIRED: The API key associated with TCPIPUtils
+      # * "URL"      Alternate url for testing.  Defaults to  "https://www.utlsapi.com/api.php?version=1.0&apikey="
+      #
+      # === Example Instantiation
+      #
+      #   options = {
+      #     :debug => true,
+      #     "APIKEY" => "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+      #     "URL" =>  "https://www.utlsapi.com/api.php?version=1.0&apikey="
+      #   }
+      #
+      #   PassiveDNS::Provider::TCPIPUtils.new(options)
+      #
+      def initialize(options={})
+        @debug = options[:debug] || false
+        @apikey = options["APIKEY"] || raise("#{self.class.name} requires an APIKEY.  See README.md")
+        @url = options["URL"] || "https://www.utlsapi.com/api.php?version=1.0&apikey="
+      end
+    
+      # Takes a label (either a domain or an IP address) and returns
+      # an array of PassiveDNS::PDNSResult instances with the answers to the query
+      def lookup(label, limit=nil)
+        $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
+        type = (label.match(/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/)) ? "domainneighbors" : "domainipdnshistory"
+        url = "#{@url}#{@apikey}&type=#{type}&q=#{label}"
+        recs = []
+        Timeout::timeout(240) {
+  				url = URI.parse url
+  				http = Net::HTTP.new(url.host, url.port)
+  				http.use_ssl = (url.scheme == 'https')
+  				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+  				http.verify_depth = 5
+  				request = Net::HTTP::Get.new(url.path+"?"+url.query)
+  				request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
+  				t1 = Time.now
+  				response = http.request(request)
+  				delta = (Time.now - t1).to_f
+  				reply = JSON.parse(response.body)
+          if reply["status"] and reply["status"] == "succeed"
+            question = reply["data"]["question"]
+            recs = format_recs(reply["data"], question, delta)
+          elsif reply["status"] and reply["status"] == "error"
+            raise "#{self.class.name}: error from web API: #{reply["data"]}"
+          end
+          if limit
+            recs[0,limit]
+          else
+            recs
+          end
+  			}
+  		rescue Timeout::Error => e
+  			$stderr.puts "#{self.class.name} lookup timed out: #{label}"
+  		end
+    
+      private
+    
+      # translates the data structure derived from of tcpiputils's JSON reply
+      def format_recs(reply_data, question, delta)
+        recs = []
+        reply_data.each do |key, data|
+          case key
+          when "ipv4"
+            data.each do |rec|
+              recs << PDNSResult.new(self.class.name, delta, question, rec["ip"], "A", nil, nil, rec["updatedate"], nil)
+            end
+          when "ipv6"
+            data.each do |rec|
+              recs << PDNSResult.new(self.class.name, delta, question, rec["ip"], "AAAA", nil, nil, rec["updatedate"], nil)
+            end
+          when "dns"
+            data.each do |rec|
+              recs << PDNSResult.new(self.class.name, delta, question, rec["dns"], "NS", nil, nil, rec["updatedate"], nil)
+            end
+          when "mx"
+            data.each do |rec|
+              recs << PDNSResult.new(self.class.name, delta, question, rec["dns"], "MX", nil, nil, rec["updatedate"], nil)
+            end
+          when "domains"
+            data.each do |rec|
+              recs << PDNSResult.new(self.class.name, delta, rec, question, "A", nil, nil, nil, nil)
+            end
+          end
+        end
+        recs
+      end
+
+    end
+  end
+end

data/lib/passivedns/client/provider/virustotal.rb

@@ -0,0 +1,105 @@
+# DESCRIPTION: this is a module for pdns.rb, primarily used by pdnstool.rb, to query VirusTotal's passive DNS database
+require 'net/http'
+require 'net/https'
+require 'openssl'
+
+module PassiveDNS #:nodoc: don't document this
+  # The Provider module contains all the Passive DNS provider client code
+  module Provider
+    # Queries VirusTotal's passive DNS database
+  	class VirusTotal < PassiveDB
+      # Sets the modules self-reported name to "VirusTotal"
+      def self.name
+        "VirusTotal"
+      end
+      # Sets the configuration section name to "virustotal"
+      def self.config_section_name
+        "virustotal"
+      end
+      # Sets the command line database argument to "v"
+      def self.option_letter
+        "v"
+      end
+    
+      # :debug enables verbose logging to standard output
+      attr_accessor :debug
+      
+      # === Options
+      # * :debug     Sets the debug flag for the module
+      # * "APIKEY"   Mandatory.  API Key associated with your VirusTotal account
+      # * "URL"      Alternate url for testing.  Defaults to https://www.virustotal.com/vtapi/v2/
+      #
+      # === Example Instantiation
+      #
+      #   options = {
+      #     :debug => true,
+      #     "APIKEY" => "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+      #     "URL" => "https://www.virustotal.com/vtapi/v2/"
+      #   }
+      #
+      #   PassiveDNS::Provider::VirusTotal.new(options)
+      #
+  		def initialize(options={})
+        @debug = options[:debug] || false
+        @apikey = options["APIKEY"] || raise("#{self.class.name} requires an APIKEY.  See README.md")
+        @url = options["URL"] || "https://www.virustotal.com/vtapi/v2/"
+      end
+
+      # Takes a label (either a domain or an IP address) and returns
+      # an array of PassiveDNS::PDNSResult instances with the answers to the query
+  		def lookup(label, limit=nil)
+  			$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
+  			Timeout::timeout(240) {
+  				url = nil
+  				if label =~ /^[\d\.]+$/
+  					url = "#{@url}ip-address/report?ip=#{label}&apikey=#{@apikey}"
+  				else
+  					url = "#{@url}domain/report?domain=#{label}&apikey=#{@apikey}"
+  				end
+  				$stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
+  				url = URI.parse url
+  				http = Net::HTTP.new(url.host, url.port)
+  				http.use_ssl = (url.scheme == 'https')
+  				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+  				http.verify_depth = 5
+  				request = Net::HTTP::Get.new(url.path+"?"+url.query)
+  				request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
+  				t1 = Time.now
+  				response = http.request(request)
+  				t2 = Time.now
+  				recs = parse_json(response.body, label, t2-t1)
+          if limit
+            recs[0,limit]
+          else
+            recs
+          end
+  			}
+  		rescue Timeout::Error => e
+  			$stderr.puts "#{self.class.name} lookup timed out: #{label}"
+  		end
+    
+      private
+    
+      # parses the response of virustotal's JSON reply to generate an array of PDNSResult
+  		def parse_json(page,query,response_time=0)
+  			res = []
+  			# need to remove the json_class tag or the parser will crap itself trying to find a class to align it to
+  			data = JSON.parse(page)
+  			if data['resolutions']
+  				data['resolutions'].each do |row|
+  					if row['ip_address']
+  						res << PDNSResult.new(self.class.name,response_time,query,row['ip_address'],'A',nil,nil,row['last_resolved'])
+  					elsif row['hostname']
+  						res << PDNSResult.new(self.class.name,response_time,row['hostname'],query,'A',nil,nil,row['last_resolved'])
+  					end
+  				end
+  			end
+  			res
+  		rescue Exception => e
+  			$stderr.puts "VirusTotal Exception: #{e}"
+  			raise e
+  		end
+    
+  	end
+  end
+end

data/lib/passivedns/client/state.rb

@@ -2,31 +2,39 @@ require 'sqlite3'
 require 'yaml'
 require 'structformatter'
 
-module PassiveDNS
+module PassiveDNS # :nodoc:
+  # struct to hold pending entries for query
 	class PDNSQueueEntry < Struct.new(:query, :state, :level); end
 
+  # holds state in memory of the queue to be queried, records returned, and the level of recursion
 	class PDNSToolState
+    # :debug enables verbose logging to standard output
 		attr_accessor :debug
+    # :level is the recursion depth
 		attr_reader :level
 
+    # creates a new, blank PDNSToolState instance
 		def initialize
 			@queue = []
 			@recs = []
 			@level = 0
 		end
-
+    
+    # returns the next record 
 		def next_result
 			@recs.each do |rec|
 				yield rec
 			end
 		end
 
+    # adds the record to the list of records received and tries to add the answer and query back to the queue for future query
 		def add_result(res)
 			@recs << res
 			add_query(res.answer,'pending')
 			add_query(res.query,'pending')
 		end
 
+    # sets the state of a given query
 		def update_query(query,state)
 			@queue.each do |q|
 				if q.query == query
@@ -37,6 +45,7 @@ module PassiveDNS
 			end
 		end
 
+    # returns the state of a provided query
 		def get_state(query)
 			@queue.each do |q|
 				if q.query == query
@@ -46,6 +55,7 @@ module PassiveDNS
 			false
 		end
 
+    # adding a query to the queue of things to be queried, but only if the query isn't already queued or answered
 		def add_query(query,state,level=@level+1)
 			if query =~ /^\d+ \w+\./
 				query = query.split(/ /,2)[1]
@@ -55,6 +65,7 @@ module PassiveDNS
 			@queue << PDNSQueueEntry.new(query,state,level)
 		end
 
+    # returns each query waiting on the queue
 		def each_query(max_level=20)
 			@queue.each do |q|
 				if q.state == 'pending' or q.state == 'failed'
@@ -67,6 +78,7 @@ module PassiveDNS
 			end
 		end
 
+    # transforms a set of results into GDF syntax
 		def to_gdf
 			output = "nodedef> name,description VARCHAR(12),color,style\n"
 			# IP "$node2,,white,1"
@@ -98,6 +110,7 @@ module PassiveDNS
 			output
 		end
 
+    # transforms a set of results into graphviz syntax
 		def to_graphviz
 			colors = {"MX" => "green", "A" => "blue", "CNAME" => "pink", "NS" => "red", "SOA" => "white", "PTR" => "purple", "TXT" => "brown"}
 			output = "graph pdns {\n"
@@ -119,6 +132,7 @@ module PassiveDNS
 			output += "}\n"
 		end
 
+    # transforms a set of results into graphml syntax
 		def to_graphml
 			output = '<?xml version="1.0" encoding="UTF-8"?>
 <graphml xmlns="http://graphml.graphdrawing.org/xmlns"  
@@ -141,6 +155,7 @@ module PassiveDNS
 			output += '</graph></graphml>'+"\n"
 		end
 
+    # transforms a set of results into XML
 		def to_xml
 			output = '<?xml version="1.0" encoding="UTF-8" ?>'+"\n"
 			output +=  "<report>\n"
@@ -152,6 +167,7 @@ module PassiveDNS
 			output +=  "</report>\n"
 		end
 
+    # transforms a set of results into YAML
 		def to_yaml
 			output = ""
 			next_result do |rec|
@@ -160,6 +176,7 @@ module PassiveDNS
 			output
 		end
 
+    # transforms a set of results into JSON
 		def to_json
 			output = "[\n"
 			sep = ""
@@ -171,6 +188,7 @@ module PassiveDNS
 			output += "\n]\n"
 		end
 
+    # transforms a set of results into a text string
 		def to_s(sep="\t")
 			output = ""
 			next_result do |rec|
@@ -181,8 +199,11 @@ module PassiveDNS
 	end # class PDNSToolState
 
 
+  # creates persistence to the tool state by leveraging SQLite3
 	class PDNSToolStateDB < PDNSToolState
 		attr_reader :level
+    # creates an SQLite3-based Passive DNS Client state
+    # only argument is the filename of the sqlite3 database
 		def initialize(sqlitedb=nil)
 			puts "PDNSToolState  initialize  #{sqlitedb}" if @debug
 			@level = 0
@@ -204,6 +225,7 @@ module PassiveDNS
 			end
 		end
 
+    # creates the sqlite3 tables needed to track the state of this tool as itqueries and recurses
 		def create_tables
 			puts "creating tables" if @debug
 			@sqlitedbh.execute("create table results (query, answer, rrtype, ttl, firstseen, lastseen, ts REAL)")
@@ -214,6 +236,7 @@ module PassiveDNS
 			@sqlitedbh.execute("create index queue_state_idx on queue (state)")
 		end
 
+    # returns the next record 
 		def next_result
 			rows = @sqlitedbh.execute("select query, answer, rrtype, ttl, firstseen, lastseen from results order by ts")
 			rows.each do |row|
@@ -221,6 +244,7 @@ module PassiveDNS
 			end
 		end
 
+    # adds the record to the list of records received and tries to add the answer and query back to the queue for future query
 		def add_result(res)
 			puts "adding result: #{res.to_s}" if @debug
 			curtime = Time.now().to_f
@@ -230,6 +254,7 @@ module PassiveDNS
 			add_query(res.query,'pending')
 		end
 
+    # adding a query to the queue of things to be queried, but only if the query isn't already queued or answered
 		def add_query(query,state,level=@level+1)
 			return if get_state(query)
 			curtime = Time.now().to_f
@@ -240,10 +265,12 @@ module PassiveDNS
 			end
 		end
 
+    # sets the state of a given query
 		def update_query(query,state)
 			@sqlitedbh.execute("update queue set state = '#{state}' where query = '#{query}'")
 		end
 
+    # returns each query waiting on the queue
 		def get_state(query)
 			rows = @sqlitedbh.execute("select state from queue where query = '#{query}'")
 			if rows
@@ -254,6 +281,7 @@ module PassiveDNS
 			false
 		end
 
+    # returns each query waiting on the queue
 		def each_query(max_level=20)
 			puts "each_query max_level=#{max_level} curlevel=#{@level}" if @debug
 			rows = @sqlitedbh.execute("select query, state, level from queue where state = 'failed' or state = 'pending' order by level limit 1")

data/lib/passivedns/client/version.rb

@@ -1,5 +1,7 @@
-module PassiveDNS
+module PassiveDNS # :nodoc:
+  # coodinates the lookups accross all configured PassiveDNS providers
   class Client
-    VERSION = "2.0.6"
+    # version of PassiveDNS::Client
+    VERSION = "2.1.0"
   end
 end

data/test/test_cli.rb

@@ -13,22 +13,23 @@ require_relative '../lib/passivedns/client/cli.rb'
 class TestCLI < Minitest::Test
   def test_letter_map
     letter_map = PassiveDNS::CLI.get_letter_map
-    assert_equal("3bcdmptv", letter_map.keys.sort.join(""))
+    assert_equal("3bcdmprtv", letter_map.keys.sort.join(""))
   end
   
   def test_help_text
     helptext = PassiveDNS::CLI.run(["--help"])
     helptext.gsub!(/Usage: .*?\[/, "Usage: [")
     assert_equal(
-"Usage: [-d [3bcdmptv]] [-g|-v|-m|-c|-x|-y|-j|-t] [-os <sep>] [-f <file>] [-r#|-w#|-v] [-l <count>] <ip|domain|cidr>
+"Usage: [-d [3bcdmprtv]] [-g|-v|-m|-c|-x|-y|-j|-t] [-os <sep>] [-f <file>] [-r#|-w#|-v] [-l <count>] <ip|domain|cidr>
 Passive DNS Providers
-  -d3bcdmptv uses all of the available passive dns database
+  -d3bcdmprtv uses all of the available passive dns database
   -d3 use 360.cn
   -db use BFK.de
   -dc use CIRCL
   -dd use DNSDB
   -dm use Mnemonic
   -dp use PassiveTotal
+  -dr use RiskIQ
   -dt use TCPIPUtils
   -dv use VirusTotal
   -dvt uses VirusTotal and TCPIPUtils (for example)
@@ -84,8 +85,8 @@ Getting Help
     assert_equal(options_target, options)
     assert_equal([], items)
    
-    options_target[:pdnsdbs] = ["circl", "dnsdb", "mnemonic"]
-    options, items = PassiveDNS::CLI.parse_command_line(["-dcdm"])
+    options_target[:pdnsdbs] = ["circl", "dnsdb", "mnemonic", "riskiq"]
+    options, items = PassiveDNS::CLI.parse_command_line(["-dcdmr"])
     assert_equal(options_target, options)
     assert_equal([], items)