passivedns-client 2.1.7 → 2.1.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 893747659885f41d56357637504abe671f95f102
-  data.tar.gz: 6f2a91db26389c0286439a6c5b82ca9dbc535a74
+SHA256:
+  metadata.gz: 8c2b55f21b59fb20d9167e1893d86aa500e3d020932d7b6e3fe5f2eed7e3113b
+  data.tar.gz: 7a48256ee4739a8a432a4a5f7aff6c7a9d8fcd1e5922b07caaf6d3557ffa5ca5
 SHA512:
-  metadata.gz: 825f792a8082db19cf426ca8d87c32245ab0e5a8d9d3ee7e6e1442030a2976549affccc2212e71a53339fdb974bd20b5c31b08df3c23423714168753bb126e6d
-  data.tar.gz: b2be17572348b7166bd567009653e5023ee1c0a142f78cb37c355fbe0c6ec38c7c7a89d32015b5f7dd6061ef85a3e152b2b8c62182e9af53c185b8076fd38e10
+  metadata.gz: e75a301a616818187ba3ea036b2ae39b8b3b562c37115cf4b6d1ece009748838d76477ef58b353f654a81e890c1f5aaa2aeb151b45c23cb5bec692845f52f732
+  data.tar.gz: 6a549456ebbc19920f00129ee0371742790f288324ade26441349cd407d9e7f8f42e49ecaac49da3b59ed5ba36267fb925d77cfc33e996d58df4d8284a28f5d2

data/README.md

@@ -2,14 +2,10 @@
 
 This rubygem queries the following Passive DNS databases: 
 
-* BFK.de
 * CIRCL
 * DNSDB (FarSight)
-* Mnemonic
-* PassiveDNS.cn (Qihoo 360 Technology Co.,Ltd)
 * PassiveTotal
 * RiskIQ
-* TCPIPUtils
 * VirusTotal
 
 Passive DNS is a technique where IP to hostname mappings are made by recording the answers of other people's queries.  
@@ -40,16 +36,8 @@ From version 2.0.0 on, all configuration keys for passive DNS providers are in o
 
 	[dnsdb]
 	APIKEY = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-	[cn360]
-	API = http://some.web.address.for.their.api
-	API_ID = a username that is given when you register
-	API_KEY = a long and random password of sorts that is used along with the page request to generate a per page API key
-	[tcpiputils]
-	APIKEY = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
 	[virustotal]
 	APIKEY = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-	[mnemonic]
-	APIKEY = 01234567890abcdef01234567890abcdef012345
 	[passivetotal]
 	USERNAME = tom@example.com
 	APIKEY = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
@@ -66,58 +54,55 @@ CIRCL also can use and authorization token.  In that case, you should drop the U
 	AUTH_TOKEN = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
 
 ## Getting Access
-* 360.cn : http://www.passivedns.cn
-* BFK.de : No registration required, but please, please ready their usage policy at http://www.bfk.de/bfk_dnslogger.html
 * CIRCL : https://www.circl.lu/services/passive-dns/
 * DNSDB (Farsight Security) : https://api.dnsdb.info/
-* Mnemonic : mss .at. mnemonic.no
 * PassiveTotal : https://www.passivetotal.org
 * RiskIQ : https://github.com/RiskIQ/python_api/blob/master/LICENSE
-* TCPIPUtils : http://www.tcpiputils.com/premium-access
 * VirusTotal : https://www.virustotal.com
 
 ## Usage
 
 	require 'passivedns/client'
 	
-	c = PassiveDNS::Client.new(['bfk','dnsdb']) # providers: bfk, tcpiputils, certee, dnsdb, virustotal, passivedns.cn, mnemonic
+	c = PassiveDNS::Client.new(['riskiq','dnsdb'])
 	results = c.query("example.com")
 	
 
 Or use the included tool...
 
-	Usage: bin/pdnstool [-d [3bcdmptv]] [-g|-v|-m|-c|-x|-y|-j|-t] [-os <sep>] [-f <file>] [-r#|-w#|-v] [-l <count>] <ip|domain|cidr>
-	Passive DNS Providers  -d3bcdmptv uses all of the available passive dns database
-	  -d3 use 360.cn
-	  -db use BFK.de
-	  -dc use CIRCL
-	  -dd use DNSDB
-	  -dm use Mnemonic
-	  -dp use PassiveTotal
-	  -dr use RiskIQ
-	  -dt use TCPIPUtils
-	  -dv use VirusTotal
-	  -dvt uses VirusTotal and TCPIPUtils (for example)
-	  
-	Output Formatting
-	  -g link-nodal GDF visualization definition
-	  -z link-nodal graphviz visualization definition
-	  -m link-nodal graphml visualization definition
-	  -c CSV
-	  -x XML
-	  -y YAML
-	  -j JSON
-	  -t ASCII text (default)
-	  -s <sep> specifies a field separator for text output, default is tab
-	  
-	State and Recusion
-	  -f[file] specifies a sqlite3 database used to read the current state - useful for large result sets and generating graphs of previous runs.
-	  -r# specifies the levels of recursion to pull. **WARNING** This is quite taxing on the pDNS servers, so use judiciously (never more than 3 or so) or find yourself blocked!
-	  -w# specifies the amount of time to wait, in seconds, between queries (Default: 0)
-	  -l <count> limits the number of records returned per passive dns database queried.
-	  
-	Getting Help
-	  -v debugging information
+    Usage: bin/pdnstool [-d [cdprv]] [-g|-v|-m|-c|-x|-y|-j|-t] [-os <sep>] [-f <file>] [-r#|-w#|-v] [-l <count>] [--config <file>] <ip|domain|cidr>
+    Passive DNS Providers
+      -dcdprv uses all of the available passive dns database
+      -dc use CIRCL
+      -dd use DNSDB
+      -dp use PassiveTotal
+      -dr use RiskIQ
+      -dv use VirusTotal
+      -dvr uses VirusTotal and RiskIQ (for example)
+
+    Output Formatting
+      -g link-nodal GDF visualization definition
+      -z link-nodal graphviz visualization definition
+      -m link-nodal graphml visualization definition
+      -c CSV
+      -x XML
+      -y YAML
+      -j JSON
+      -t ASCII text (default)
+      -s <sep> specifies a field separator for text output, default is tab
+
+    State and Recursion
+      -f[file] specifies a sqlite3 database used to read the current state - useful for large result sets and generating graphs of previous runs.
+      -r# specifies the levels of recursion to pull. **WARNING** This is quite taxing on the pDNS servers, so use judiciously (never more than 3 or so) or find yourself blocked!
+      -w# specifies the amount of time to wait, in seconds, between queries (Default: 0)
+      -l <count> limits the number of records returned per passive dns database queried.
+
+    Specifying a Configuration File
+      --config <file> specifies a config file. default: /home/chris/.passivedns-client
+
+    Getting Help
+      -h hello there.  This option produces this helpful help information on how to access help.
+      -v debugging information
 
 ## Writing Your Own Database Adaptor
 

data/lib/passivedns/client.rb

@@ -92,6 +92,12 @@ module PassiveDNS # :nodoc:
       end
     end
     
+    def timeout=(t)
+      @pdnsdbs.each do |pdnsdb|
+        pdnsdb.timeout = t
+      end
+    end
+    
     # perform the query lookup accross all configured PassiveDNS providers
     def query(item, limit=nil)
       threads = []

data/lib/passivedns/client/cli.rb

@@ -58,7 +58,8 @@ module PassiveDNS # :nodoc:
         [ '--sqlite3', '-f', GetoptLong::REQUIRED_ARGUMENT ],
         [ '--recurse', '-r', GetoptLong::REQUIRED_ARGUMENT ],
         [ '--wait', '-w', GetoptLong::REQUIRED_ARGUMENT ],
-        [ '--limit', '-l', GetoptLong::REQUIRED_ARGUMENT ]
+        [ '--limit', '-l', GetoptLong::REQUIRED_ARGUMENT ],
+        [ '--config', GetoptLong::REQUIRED_ARGUMENT ]
       )
 
       letter_map = get_letter_map
@@ -74,7 +75,8 @@ module PassiveDNS # :nodoc:
         :debug => false,
         :sqlitedb => nil,
         :limit => nil,
-        :help => false
+        :help => false,
+        :configfile => "#{ENV['HOME']}/.passivedns-client"
       }
 
       opts.each do |opt, arg|
@@ -121,6 +123,8 @@ module PassiveDNS # :nodoc:
           options[:sqlitedb] = arg
         when '--limit'
           options[:limit] = arg.to_i
+        when '--config'
+          options[:configfile] = arg
         else
           options[:help] = true
         end
@@ -129,12 +133,7 @@ module PassiveDNS # :nodoc:
       ARGV.replace(origARGV)
 
       if options[:pdnsdbs].length == 0
-        options[:pdnsdbs] << "bfk"
-      end
-
-      if options[:pdnsdbs].index("bfk") and options[:recursedepth] > 1 and options[:wait] < 60
-        options[:wait] = 60
-        $stderr.puts "Enforcing a minimal 60 second wait when using BFK for recursive crawling"
+        options[:pdnsdbs] << "virustotal"
       end
 
       if options[:debug]
@@ -158,13 +157,13 @@ module PassiveDNS # :nodoc:
     def self.usage(letter_map)
       databases = letter_map.keys.sort.join("")
       help_text = ""
-      help_text << "Usage: #{$0} [-d [#{databases}]] [-g|-v|-m|-c|-x|-y|-j|-t] [-os <sep>] [-f <file>] [-r#|-w#|-v] [-l <count>] <ip|domain|cidr>\n"
+      help_text << "Usage: #{$0} [-d [#{databases}]] [-g|-v|-m|-c|-x|-y|-j|-t] [-os <sep>] [-f <file>] [-r#|-w#|-v] [-l <count>] [--config <file>] <ip|domain|cidr>\n"
       help_text << "Passive DNS Providers\n"
       help_text << "  -d#{databases} uses all of the available passive dns database\n"
       letter_map.keys.sort.each do |l|
         help_text << "  -d#{l} use #{letter_map[l][0]}\n"
       end
-      help_text << "  -dvt uses VirusTotal and TCPIPUtils (for example)\n"
+      help_text << "  -dvr uses VirusTotal and RiskIQ (for example)\n"
       help_text << "\n"
       help_text << "Output Formatting\n"
       help_text << "  -g link-nodal GDF visualization definition\n"
@@ -183,6 +182,9 @@ module PassiveDNS # :nodoc:
       help_text << "  -w# specifies the amount of time to wait, in seconds, between queries (Default: 0)\n"
       help_text << "  -l <count> limits the number of records returned per passive dns database queried.\n"
       help_text << "\n"
+      help_text << "Specifying a Configuration File\n"
+      help_text << "  --config <file> specifies a config file. default: #{ENV['HOME']}/.passivedns-client\n"
+      help_text << "\n"
       help_text << "Getting Help\n"
       help_text << "  -h hello there.  This option produces this helpful help information on how to access help.\n"
       help_text << "  -v debugging information\n"
@@ -249,6 +251,7 @@ module PassiveDNS # :nodoc:
       else
         state = PassiveDNS::PDNSToolState.new
       end
+      state
     end
     
     # main method, takes command-line arguments and performs the desired queries and outputs
@@ -264,7 +267,7 @@ module PassiveDNS # :nodoc:
       state = create_state(options[:sqlitedb])
       state.debug = options[:debug]
 
-      pdnsclient = PassiveDNS::Client.new(options[:pdnsdbs])
+      pdnsclient = PassiveDNS::Client.new(options[:pdnsdbs], options[:configfile])
       pdnsclient.debug = options[:debug]
       
       if items.length > 0

data/lib/passivedns/client/provider/circl.rb

@@ -47,6 +47,7 @@ module PassiveDNS #:nodoc: don't document this
       #
       def initialize(options={})
         @debug = options[:debug] || false
+        @timeout = options[:timeout] || 20
         @username = options["USERNAME"]
         @password = options["PASSWORD"]
         @auth_token = options["AUTH_TOKEN"]
@@ -57,10 +58,16 @@ module PassiveDNS #:nodoc: don't document this
       # an array of PassiveDNS::PDNSResult instances with the answers to the query
       def lookup(label, limit=nil)
         $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
-        Timeout::timeout(240) {
+        recs = []
+        Timeout::timeout(@timeout) {
           url = @url+"/"+label
           $stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
-          url = URI.parse url
+          begin
+            url = URI.parse url
+          rescue URI::InvalidURIError
+            $stderr.puts "ERROR: Invalid address: #{url}"
+            return recs
+          end
           http = Net::HTTP.new(url.host, url.port)
           http.use_ssl = (url.scheme == 'https')
           http.verify_mode = OpenSSL::SSL::VERIFY_NONE
@@ -74,17 +81,26 @@ module PassiveDNS #:nodoc: don't document this
             request.add_field("Authorization", @auth_token)
           end
           t1 = Time.now
-          response = http.request(request)
-          t2 = Time.now
-          recs = parse_json(response.body, label, t2-t1)
+          0.upto(9) do
+            response = http.request(request)
+            body = response.body
+            if body == "Rate Limit Exceeded"
+              $stderr.puts "DEBUG: Rate Limit Exceeded. Retrying #{label}" if @debug
+            else
+              t2 = Time.now
+              recs  = parse_json(response.body, label, t2-t1)
+              break
+            end
+          end
           if limit
             recs[0,limit]
           else
             recs
           end
         }
-      rescue Timeout::Error => e
+      rescue Timeout::Error
         $stderr.puts "#{self.class.name} lookup timed out: #{label}"
+        recs
       end
 
       private

data/lib/passivedns/client/provider/dnsdb.rb

@@ -41,6 +41,7 @@ module PassiveDNS #:nodoc: don't document this
       #
       def initialize(options={})
         @debug = options[:debug] || false
+        @timeout = options[:timeout] || 20
         @key = options["APIKEY"] || raise("APIKEY option required for #{self.class}")
         @base = options["URL"] || "https://api.dnsdb.info/lookup"
       end
@@ -49,7 +50,7 @@ module PassiveDNS #:nodoc: don't document this
       # an array of PassiveDNS::PDNSResult instances with the answers to the query
       def lookup(label, limit=nil)
         $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
-        Timeout::timeout(240) {
+        Timeout::timeout(@timeout) {
           url = nil
           if label =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(\/\d{1,2})?$/
             label = label.gsub(/\//,',')
@@ -76,7 +77,7 @@ module PassiveDNS #:nodoc: don't document this
           $stderr.puts response.body if @debug
           parse_json(response.body,t2-t1)
         }
-      rescue Timeout::Error => e
+      rescue Timeout::Error
         $stderr.puts "#{self.class.name} lookup timed out: #{label}"
       end
       

data/lib/passivedns/client/provider/passivetotal.rb

@@ -54,6 +54,7 @@ module PassiveDNS #:nodoc: don't document this
       #
       def initialize(options={})
         @debug = options[:debug] || false
+        @timeout = options[:timeout] || 20
         @username = options["USERNAME"] || raise("#{self.class.name} requires a USERNAME")
         @apikey = options["APIKEY"] || raise("#{self.class.name} requires an APIKEY")
         @url = options["URL"] || "https://api.passivetotal.org/v2/dns/passive"
@@ -63,7 +64,7 @@ module PassiveDNS #:nodoc: don't document this
       # an array of PassiveDNS::PDNSResult instances with the answers to the query
       def lookup(label, limit=nil)
         $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
-        Timeout::timeout(240) {
+        Timeout::timeout(@timeout) {
           url = @url+"?query=#{label}"
           $stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
           url = URI.parse url
@@ -85,7 +86,7 @@ module PassiveDNS #:nodoc: don't document this
             recs
           end
         }
-      rescue Timeout::Error => e
+      rescue Timeout::Error
         $stderr.puts "#{self.class.name} lookup timed out: #{label}"
       end
     
@@ -93,8 +94,12 @@ module PassiveDNS #:nodoc: don't document this
     
       # parses the response of passivetotals's JSON reply to generate an array of PDNSResult
       def parse_json(page,query,response_time=0)
-         res = []
+        res = []
         data = JSON.parse(page)
+        pp data
+        if data['message']
+          raise "#{self.class.name} Error: #{data['message']}"
+        end
         query = data['queryValue']
         if data['results']
           data['results'].each do |row|

data/lib/passivedns/client/provider/riskiq.rb

@@ -46,10 +46,11 @@ module PassiveDNS #:nodoc: don't document this
       #
       def initialize(options={})
         @debug = options[:debug] || false
+        @timeout = options[:timeout] || 20
         @token = options["API_TOKEN"] || raise("#{self.class.name} requires an API_TOKEN")
         @privkey = options["API_PRIVATE_KEY"] || raise("#{self.class.name} requires an API_PRIVATE_KEY")
-        @server = options["API_SERVER"] || "ws.riskiq.net"
         @version = options["API_VERSION"] || "v1"
+        @server = options["API_SERVER"] || api_settings[@version][:server]
         @url = "https://#{@server}/#{@version}"
       end
 
@@ -57,7 +58,7 @@ module PassiveDNS #:nodoc: don't document this
       # an array of PassiveDNS::PDNSResult instances with the answers to the query
       def lookup(label, limit=nil)
         $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
-        Timeout::timeout(240) {
+        Timeout::timeout(@timeout) {
           url = nil
           params = {"rrType" => "", "maxResults" => limit || 1000}
         
@@ -65,8 +66,10 @@ module PassiveDNS #:nodoc: don't document this
             url = @url+"/dns/data"
             params["ip"] = label 
           else
-            url = @url+"/dns/name"
-            params["name"] = label
+            resource = api_settings[@version][:resource]
+            param = api_settings[@version][:param]
+            url = @url+"/dns/#{resource}"
+            params[param] = label
           end
           url << "?"
           params.each do |k,v|
@@ -95,16 +98,29 @@ module PassiveDNS #:nodoc: don't document this
             recs
           end
         }
-      rescue Timeout::Error => e
+      rescue Timeout::Error
         $stderr.puts "#{self.class.name} lookup timed out: #{label}"
       end
     
       private
     
+      def api_settings
+        @api_settings ||= {
+          'v1' => { server: "ws.riskiq.net", resource: 'name', param: 'name' },
+          'v2' => { server: "api.passivetotal.org", resource: 'passive', param: 'query' }
+        }
+      end
+
       # parses the response of riskiq's JSON reply to generate an array of PDNSResult
       def parse_json(page,query,response_time=0)
-         res = []
+        res = []
         data = JSON.parse(page)
+        if data['message']
+          if data['message'] =~ /quota_exceeded/
+            $stderr.puts "ERROR: quota exceeded."
+            return res
+          end
+        end
         if data['records']
           data['records'].each do |record|
             name = record['name'].gsub!(/\.$/,'')

data/lib/passivedns/client/provider/virustotal.rb

@@ -41,6 +41,7 @@ module PassiveDNS #:nodoc: don't document this
       #
       def initialize(options={})
         @debug = options[:debug] || false
+        @timeout = options[:timeout] || 20
         @apikey = options["APIKEY"] || raise("#{self.class.name} requires an APIKEY.  See README.md")
         @url = options["URL"] || "https://www.virustotal.com/vtapi/v2/"
       end
@@ -49,7 +50,7 @@ module PassiveDNS #:nodoc: don't document this
       # an array of PassiveDNS::PDNSResult instances with the answers to the query
       def lookup(label, limit=nil)
         $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
-        Timeout::timeout(240) {
+        Timeout::timeout(@timeout) {
           url = nil
           if label =~ /^[\d\.]+$/
             url = "#{@url}ip-address/report?ip=#{label}&apikey=#{@apikey}"
@@ -57,7 +58,12 @@ module PassiveDNS #:nodoc: don't document this
             url = "#{@url}domain/report?domain=#{label}&apikey=#{@apikey}"
           end
           $stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
-          url = URI.parse url
+          begin
+            url = URI.parse url
+          rescue URI::InvalidURIError
+            $stderr.puts "ERROR: Invalid address: #{url}"
+            return
+          end
           http = Net::HTTP.new(url.host, url.port)
           http.use_ssl = (url.scheme == 'https')
           http.verify_mode = OpenSSL::SSL::VERIFY_NONE
@@ -66,6 +72,10 @@ module PassiveDNS #:nodoc: don't document this
           request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
           t1 = Time.now
           response = http.request(request)
+          if response.code.to_i == 204
+            $stderr.puts "DEBUG: empty response from server" if @debug
+            return
+          end
           t2 = Time.now
           recs = parse_json(response.body, label, t2-t1)
           if limit
@@ -74,7 +84,7 @@ module PassiveDNS #:nodoc: don't document this
             recs
           end
         }
-      rescue Timeout::Error => e
+      rescue Timeout::Error
         $stderr.puts "#{self.class.name} lookup timed out: #{label}"
       end
     
@@ -83,6 +93,7 @@ module PassiveDNS #:nodoc: don't document this
       # parses the response of virustotal's JSON reply to generate an array of PDNSResult
       def parse_json(page,query,response_time=0)
         res = []
+        return res if !page
         data = JSON.parse(page)
         if data['resolutions']
           data['resolutions'].each do |row|
@@ -94,6 +105,9 @@ module PassiveDNS #:nodoc: don't document this
             end
           end
         end
+        if data['response_code'] == 0
+          $stderr.puts "DEBUG: server returned error: #{data['verbose_msg']}" if @debug
+        end
         res
       rescue Exception => e
         $stderr.puts "VirusTotal Exception: #{e}"