passivedns-client 2.1.9 → 2.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2b6b0dc4227f010b9fc53b01f2307f49b405db80
-  data.tar.gz: 1e3b0abb99ceddee325ee4245f9924500127089d
+  metadata.gz: 41bb1d54d9a49eb1995dfdd9c78538dc91ed40d7
+  data.tar.gz: c446107f1d435fcc1334b20d62c72ec18c495a3f
 SHA512:
-  metadata.gz: d638e8dba5e9af5476c626dca055925c702bd41c3b836ba876d914d8d2e609a9ed042983da8987b0d1ae7af41316c1ad5d3a4f559f5db2bf694a96d2bae06547
-  data.tar.gz: 22b0c684739e932b646f2bd7a508f9e8a53c4c2ff8ba69c5623f36a51d7d4931f83d317dcc2fbab9df3b8944f84101c854596c32a6c07486adaaad445404c028
+  metadata.gz: 4117587bbb9789a381696295a40d88fff86e0284e358ec8feeae64f1436de6f09d17db8f71f3add3d950f8fd6f4ce31b43bd19b7f8a2ff1fabd4e5a4d5e135b9
+  data.tar.gz: da94f1292023be6bb0ec75a380ac3fa661e8ecfbe6cb2fc13ad327a9930a28d7cd23b81ab1d6715df817b85760ddaab14f90fd07c4bdb16634fab521de6a8823

data/README.md

@@ -115,6 +115,9 @@ Or use the included tool...
 	  -r# specifies the levels of recursion to pull. **WARNING** This is quite taxing on the pDNS servers, so use judiciously (never more than 3 or so) or find yourself blocked!
 	  -w# specifies the amount of time to wait, in seconds, between queries (Default: 0)
 	  -l <count> limits the number of records returned per passive dns database queried.
+    
+	Specifying a Configuration File
+	  --config <file> specifies a config file. default: #{ENV['HOME']}/.passivedns-client
 	  
 	Getting Help
 	  -v debugging information

data/lib/passivedns/client.rb

@@ -94,6 +94,12 @@ module PassiveDNS # :nodoc:
       end
     end
     
+    def timeout=(t)
+      @pdnsdbs.each do |pdnsdb|
+        pdnsdb.timeout = t
+      end
+    end
+    
     # perform the query lookup accross all configured PassiveDNS providers
     def query(item, limit=nil)
       threads = []

data/lib/passivedns/client/cli.rb

@@ -75,7 +75,8 @@ module PassiveDNS # :nodoc:
         :debug => false,
         :sqlitedb => nil,
         :limit => nil,
-        :help => false
+        :help => false,
+        :configfile => "#{ENV['HOME']}/.passivedns-client"
       }
 
       opts.each do |opt, arg|
@@ -186,6 +187,9 @@ module PassiveDNS # :nodoc:
       help_text << "  -w# specifies the amount of time to wait, in seconds, between queries (Default: 0)\n"
       help_text << "  -l <count> limits the number of records returned per passive dns database queried.\n"
       help_text << "\n"
+      help_text << "Specifying a Configuration File\n"
+      help_text << "  --config <file> specifies a config file. default: #{ENV['HOME']}/.passivedns-client\n"
+      help_text << "\n"
       help_text << "Getting Help\n"
       help_text << "  -h hello there.  This option produces this helpful help information on how to access help.\n"
       help_text << "  -v debugging information\n"

data/lib/passivedns/client/provider/bfk.rb

@@ -36,6 +36,7 @@ module PassiveDNS #:nodoc: don't document this
       #
       def initialize(options={})
         @debug = options[:debug] || false
+        @timeout = options[:timeout] || 20
         @base = options["URL"] || "http://www.bfk.de/bfk_dnslogger.html?query="
         raise "Due to the EU GDPR policy, this service has been shut down until further notice."
       end
@@ -44,7 +45,7 @@ module PassiveDNS #:nodoc: don't document this
       # an array of PassiveDNS::PDNSResult instances with the answers to the query
        def lookup(label, limit=nil)  
         $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
-        Timeout::timeout(240) {
+        Timeout::timeout(@timeout) {
           t1 = Time.now
           open(
             @base+label,

data/lib/passivedns/client/provider/circl.rb

@@ -47,6 +47,7 @@ module PassiveDNS #:nodoc: don't document this
       #
       def initialize(options={})
         @debug = options[:debug] || false
+        @timeout = options[:timeout] || 20
         @username = options["USERNAME"]
         @password = options["PASSWORD"]
         @auth_token = options["AUTH_TOKEN"]
@@ -57,10 +58,16 @@ module PassiveDNS #:nodoc: don't document this
       # an array of PassiveDNS::PDNSResult instances with the answers to the query
       def lookup(label, limit=nil)
         $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
-        Timeout::timeout(240) {
+        recs = []
+        Timeout::timeout(@timeout) {
           url = @url+"/"+label
           $stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
-          url = URI.parse url
+          begin
+            url = URI.parse url
+          rescue URI::InvalidURIError
+            $stderr.puts "ERROR: Invalid address: #{url}"
+            return recs
+          end
           http = Net::HTTP.new(url.host, url.port)
           http.use_ssl = (url.scheme == 'https')
           http.verify_mode = OpenSSL::SSL::VERIFY_NONE
@@ -74,9 +81,17 @@ module PassiveDNS #:nodoc: don't document this
             request.add_field("Authorization", @auth_token)
           end
           t1 = Time.now
-          response = http.request(request)
-          t2 = Time.now
-          recs = parse_json(response.body, label, t2-t1)
+          0.upto(9) do
+            response = http.request(request)
+            body = response.body
+            if body == "Rate Limit Exceeded"
+              $stderr.puts "DEBUG: Rate Limit Exceeded. Retrying #{label}" if @debug
+            else
+              t2 = Time.now
+              recs  = parse_json(response.body, label, t2-t1)
+              break
+            end
+          end
           if limit
             recs[0,limit]
           else
@@ -85,6 +100,7 @@ module PassiveDNS #:nodoc: don't document this
         }
       rescue Timeout::Error => e
         $stderr.puts "#{self.class.name} lookup timed out: #{label}"
+        recs
       end
 
       private

data/lib/passivedns/client/provider/cn360.rb

@@ -43,6 +43,7 @@ module PassiveDNS #:nodoc: don't document this
       #
       def initialize(options={})
         @debug = options[:debug] || false
+        @timeout = options[:timeout] || 20
         ["API", "API_ID", "API_KEY"].each do |opt|
           if not options[opt]
             raise "Field #{opt} is required.  See README.md"
@@ -60,28 +61,33 @@ module PassiveDNS #:nodoc: don't document this
         end
         limit ||= 10000
         path = "/api/#{table}/keyword/#{label}/count/#{limit}/"
-        url = @cp["API"]+path
-        url = URI.parse url
-        http = Net::HTTP.new(url.host, url.port)
-        http.use_ssl = (url.scheme == 'https')
-        http.verify_mode = OpenSSL::SSL::VERIFY_NONE # I hate doing this
-        http.verify_depth = 5
-        request = Net::HTTP::Get.new(url.path)
-        request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
-        request.add_field('Accept', 'application/json')
-        request.add_field("X-BashTokid", @cp["API_ID"])
-        token = Digest::MD5.hexdigest(path+@cp["API_KEY"])
-        $stderr.puts "DEBUG: cn360 url = #{url} token = #{token}" if @debug
-        request.add_field("X-BashToken", token)
-        t1 = Time.now
-        response = http.request(request)
-        t2 = Time.now
-        recs = parse_json(response.body, label, t2-t1)
-        if limit
-          recs[0,limit]
-        else
-          recs
-        end
+        Timeout::timeout(@timeout) {
+          url = @cp["API"]+path
+          url = URI.parse url
+          http = Net::HTTP.new(url.host, url.port)
+          http.use_ssl = (url.scheme == 'https')
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE # I hate doing this
+          http.verify_depth = 5
+          request = Net::HTTP::Get.new(url.path)
+          request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
+          request.add_field('Accept', 'application/json')
+          request.add_field("X-BashTokid", @cp["API_ID"])
+          token = Digest::MD5.hexdigest(path+@cp["API_KEY"])
+          $stderr.puts "DEBUG: cn360 url = #{url} token = #{token}" if @debug
+          request.add_field("X-BashToken", token)
+          t1 = Time.now
+          response = http.request(request)
+          t2 = Time.now
+          recs = parse_json(response.body, label, t2-t1)
+          if limit
+            recs[0,limit]
+          else
+            recs
+          end
+        }
+      rescue Timeout::Error => e
+        $stderr.puts "#{self.class.name} lookup timed out: #{label}"
+        recs
       end
       
       private

data/lib/passivedns/client/provider/dnsdb.rb

@@ -41,6 +41,7 @@ module PassiveDNS #:nodoc: don't document this
       #
       def initialize(options={})
         @debug = options[:debug] || false
+        @timeout = options[:timeout] || 20
         @key = options["APIKEY"] || raise("APIKEY option required for #{self.class}")
         @base = options["URL"] || "https://api.dnsdb.info/lookup"
       end
@@ -49,7 +50,7 @@ module PassiveDNS #:nodoc: don't document this
       # an array of PassiveDNS::PDNSResult instances with the answers to the query
       def lookup(label, limit=nil)
         $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
-        Timeout::timeout(240) {
+        Timeout::timeout(@timeout) {
           url = nil
           if label =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(\/\d{1,2})?$/
             label = label.gsub(/\//,',')

data/lib/passivedns/client/provider/mnemonic.rb

@@ -41,6 +41,7 @@ module PassiveDNS #:nodoc: don't document this
       #
       def initialize(options={})
         @debug = options[:debug] || false
+        @timeout = options[:timeout] || 20
         @apikey = options["APIKEY"]
         @url = options["URL"] || "https://api.mnemonic.no/pdns/v3/"
         if @url == "https://passivedns.mnemonic.no/api1/?apikey="
@@ -52,7 +53,7 @@ module PassiveDNS #:nodoc: don't document this
       # an array of PassiveDNS::PDNSResult instances with the answers to the query
       def lookup(label, limit=nil)
         $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
-        Timeout::timeout(240) {
+        Timeout::timeout(@timeout) {
           url = "#{@url}#{label}"
           $stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
           url = URI.parse url

data/lib/passivedns/client/provider/passivetotal.rb

@@ -54,6 +54,7 @@ module PassiveDNS #:nodoc: don't document this
       #
       def initialize(options={})
         @debug = options[:debug] || false
+        @timeout = options[:timeout] || 20
         @username = options["USERNAME"] || raise("#{self.class.name} requires a USERNAME")
         @apikey = options["APIKEY"] || raise("#{self.class.name} requires an APIKEY")
         @url = options["URL"] || "https://api.passivetotal.org/v2/dns/passive"
@@ -63,7 +64,7 @@ module PassiveDNS #:nodoc: don't document this
       # an array of PassiveDNS::PDNSResult instances with the answers to the query
       def lookup(label, limit=nil)
         $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
-        Timeout::timeout(240) {
+        Timeout::timeout(@timeout) {
           url = @url+"?query=#{label}"
           $stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
           url = URI.parse url

data/lib/passivedns/client/provider/riskiq.rb

@@ -46,10 +46,11 @@ module PassiveDNS #:nodoc: don't document this
       #
       def initialize(options={})
         @debug = options[:debug] || false
+        @timeout = options[:timeout] || 20
         @token = options["API_TOKEN"] || raise("#{self.class.name} requires an API_TOKEN")
         @privkey = options["API_PRIVATE_KEY"] || raise("#{self.class.name} requires an API_PRIVATE_KEY")
-        @server = options["API_SERVER"] || "ws.riskiq.net"
         @version = options["API_VERSION"] || "v1"
+        @server = options["API_SERVER"] || api_settings[@version][:server]
         @url = "https://#{@server}/#{@version}"
       end
 
@@ -57,7 +58,7 @@ module PassiveDNS #:nodoc: don't document this
       # an array of PassiveDNS::PDNSResult instances with the answers to the query
       def lookup(label, limit=nil)
         $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
-        Timeout::timeout(240) {
+        Timeout::timeout(@timeout) {
           url = nil
           params = {"rrType" => "", "maxResults" => limit || 1000}
         
@@ -65,8 +66,10 @@ module PassiveDNS #:nodoc: don't document this
             url = @url+"/dns/data"
             params["ip"] = label 
           else
-            url = @url+"/dns/name"
-            params["name"] = label
+            resource = api_settings[@version][:resource]
+            param = api_settings[@version][:param]
+            url = @url+"/dns/#{resource}"
+            params[param] = label
           end
           url << "?"
           params.each do |k,v|
@@ -101,10 +104,23 @@ module PassiveDNS #:nodoc: don't document this
     
       private
     
+      def api_settings
+        @api_settings ||= {
+          'v1' => { server: "ws.riskiq.net", resource: 'name', param: 'name' },
+          'v2' => { server: "api.passivetotal.org", resource: 'passive', param: 'query' }
+        }
+      end
+
       # parses the response of riskiq's JSON reply to generate an array of PDNSResult
       def parse_json(page,query,response_time=0)
-         res = []
+        res = []
         data = JSON.parse(page)
+        if data['message']
+          if data['message'] =~ /quota_exceeded/
+            $stderr.puts "ERROR: quota exceeded."
+            return res
+          end
+        end
         if data['records']
           data['records'].each do |record|
             name = record['name'].gsub!(/\.$/,'')

data/lib/passivedns/client/provider/tcpiputils.rb

@@ -42,6 +42,7 @@ module PassiveDNS #:nodoc: don't document this
       #
       def initialize(options={})
         @debug = options[:debug] || false
+        @timeout = options[:timeout] || 20
         @apikey = options["APIKEY"] || raise("#{self.class.name} requires an APIKEY.  See README.md")
         @url = options["URL"] || "https://www.utlsapi.com/api.php?version=1.0&apikey="
       end
@@ -53,7 +54,7 @@ module PassiveDNS #:nodoc: don't document this
         type = (label.match(/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/)) ? "domainneighbors" : "domainipdnshistory"
         url = "#{@url}#{@apikey}&type=#{type}&q=#{label}"
         recs = []
-        Timeout::timeout(240) {
+        Timeout::timeout(@timeout) {
           url = URI.parse url
           http = Net::HTTP.new(url.host, url.port)
           http.use_ssl = (url.scheme == 'https')

data/lib/passivedns/client/provider/virustotal.rb

@@ -41,6 +41,7 @@ module PassiveDNS #:nodoc: don't document this
       #
       def initialize(options={})
         @debug = options[:debug] || false
+        @timeout = options[:timeout] || 20
         @apikey = options["APIKEY"] || raise("#{self.class.name} requires an APIKEY.  See README.md")
         @url = options["URL"] || "https://www.virustotal.com/vtapi/v2/"
       end
@@ -49,7 +50,7 @@ module PassiveDNS #:nodoc: don't document this
       # an array of PassiveDNS::PDNSResult instances with the answers to the query
       def lookup(label, limit=nil)
         $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
-        Timeout::timeout(240) {
+        Timeout::timeout(@timeout) {
           url = nil
           if label =~ /^[\d\.]+$/
             url = "#{@url}ip-address/report?ip=#{label}&apikey=#{@apikey}"
@@ -57,7 +58,12 @@ module PassiveDNS #:nodoc: don't document this
             url = "#{@url}domain/report?domain=#{label}&apikey=#{@apikey}"
           end
           $stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
-          url = URI.parse url
+          begin
+            url = URI.parse url
+          rescue URI::InvalidURIError
+            $stderr.puts "ERROR: Invalid address: #{url}"
+            return
+          end
           http = Net::HTTP.new(url.host, url.port)
           http.use_ssl = (url.scheme == 'https')
           http.verify_mode = OpenSSL::SSL::VERIFY_NONE
@@ -66,6 +72,10 @@ module PassiveDNS #:nodoc: don't document this
           request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
           t1 = Time.now
           response = http.request(request)
+          if response.code.to_i == 204
+            $stderr.puts "DEBUG: empty response from server" if @debug
+            return
+          end
           t2 = Time.now
           recs = parse_json(response.body, label, t2-t1)
           if limit
@@ -83,6 +93,7 @@ module PassiveDNS #:nodoc: don't document this
       # parses the response of virustotal's JSON reply to generate an array of PDNSResult
       def parse_json(page,query,response_time=0)
         res = []
+        return res if !page
         data = JSON.parse(page)
         if data['resolutions']
           data['resolutions'].each do |row|
@@ -94,6 +105,9 @@ module PassiveDNS #:nodoc: don't document this
             end
           end
         end
+        if data['response_code'] == 0
+          $stderr.puts "DEBUG: server returned error: #{data['verbose_msg']}" if @debug
+        end
         res
       rescue Exception => e
         $stderr.puts "VirusTotal Exception: #{e}"

data/lib/passivedns/client/version.rb

@@ -2,6 +2,6 @@ module PassiveDNS # :nodoc:
   # coodinates the lookups accross all configured PassiveDNS providers
   class Client
     # version of PassiveDNS::Client
-    VERSION = "2.1.9"
+    VERSION = "2.1.11"
   end
 end

data/test/test_cli.rb

@@ -51,6 +51,9 @@ State and Recursion
   -w# specifies the amount of time to wait, in seconds, between queries (Default: 0)
   -l <count> limits the number of records returned per passive dns database queried.
 
+Specifying a Configuration File
+  --config <file> specifies a config file. default: #{ENV['HOME']}/.passivedns-client
+
 Getting Help
   -h hello there.  This option produces this helpful help information on how to access help.
   -v debugging information
@@ -68,7 +71,8 @@ Getting Help
       :debug => false,
       :sqlitedb => nil,
       :limit => nil,
-      :help => false
+      :help => false,
+      :configfile => "#{ENV['HOME']}/.passivedns-client"
     }
     
     options, items = PassiveDNS::CLI.parse_command_line([])
@@ -108,7 +112,8 @@ Getting Help
       :debug => false,
       :sqlitedb => nil,
       :limit => nil,
-      :help => false
+      :help => false,
+      :configfile => "#{ENV['HOME']}/.passivedns-client"
     }
     
     options_target[:sep] = ","
@@ -171,7 +176,8 @@ Getting Help
       :debug => false,
       :sqlitedb => nil,
       :limit => nil,
-      :help => true
+      :help => true,
+      :configfile => "#{ENV['HOME']}/.passivedns-client"
     }
 
     options, items = PassiveDNS::CLI.parse_command_line(["-dptv", "-h", "8.8.8.8"])
@@ -195,7 +201,8 @@ Getting Help
       :debug => false,
       :sqlitedb => "test.db",
       :limit => 10,
-      :help => false
+      :help => false,
+      :configfile => "#{ENV['HOME']}/.passivedns-client"
     }
 
     options, items = PassiveDNS::CLI.parse_command_line(["-dptv", "-f", "test.db", "-r", "5", "-w", "30", "-l", "10", "8.8.8.8"])
@@ -215,11 +222,11 @@ Getting Help
       :sqlitedb => nil,
       :limit => nil,
       :help => false,
-      :configfile => ".passivedns-client"
+      :configfile => "#{ENV['HOME']}/.passivedns-client"
     }
     
     
-    options, items = PassiveDNS::CLI.parse_command_line(["--config", ".passivedns-client"])
+    options, items = PassiveDNS::CLI.parse_command_line(["--config", "#{ENV['HOME']}/.passivedns-client"])
     assert_equal(options_target, options)
     assert_equal([], items)
     

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: passivedns-client
 version: !ruby/object:Gem::Version
-  version: 2.1.9
+  version: 2.1.11
 platform: ruby
 authors:
 - chrislee35
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-02 00:00:00.000000000 Z
+date: 2018-06-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json