passivedns-client 2.0.1 → 2.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 87b8057605ef62a693b4460fac1e72fd8c8a0afb
-  data.tar.gz: 9adef70cf9929f665e75a781f5e3216a9a3207c9
+  metadata.gz: 9e141cbc5d84ea9c25ee8c207db9c7758869117e
+  data.tar.gz: 81c70c9a9a4c272133c4e33a2e043fab650fd00a
 SHA512:
-  metadata.gz: 43d6004ae73822197e0db5e4cfced825462820d3b626c75574cbddaea2ab3c49ba301067e3c58113972cc25008f4b7967076571c6fd1ab72fef09b5faa781c96
-  data.tar.gz: 8ba404ddc64987dae536adf248a43ac6a30c9b658c71256d5cc27aa02c2e26243b22706676f80d5e733ddd06cc92cc4990b8b8ed215da3b88d7ed001114cb3be
+  metadata.gz: fffadc8b47040b55d25255cf62ab2b8f2498a8a39d89caa1812da37944da266f411cf9b384cb1f6ab70718ac106811313a0d4987c792487919f239ad388bc4ed
+  data.tar.gz: d0592a94d6f4cec71afdf55d3cccedf1cfc1fe8ecf17cc3f3d746d92b184a694e3c1e1655c17d03f3972a63bc56de433d3cb7ec9eb067172d02183f7c7ff4a5b

data/bin/pdnstool

@@ -1,214 +1,5 @@
 #!/usr/bin/env ruby
 require 'passivedns/client'
-require 'structformatter'
-require 'getoptlong'
-require 'yaml'
+require 'passivedns/client/cli'
 
-def pdnslookup(state,pdnsclient,recursedepth=1,wait=0,debug=false,limit=nil)
-	puts "pdnslookup: #{state.level} #{recursedepth}" if debug
-	level = 0
-	while level < recursedepth
-		puts "pdnslookup: #{level} < #{recursedepth}" if debug
-		state.each_query(recursedepth) do |q|
-			rv = pdnsclient.query(q,limit)
-			if rv
-				rv.each do |r|
-					if ["A","AAAA","NS","CNAME","PTR"].index(r.rrtype)
-						puts "pdnslookup: #{r.to_s}" if debug
-						state.add_result(r)
-					end
-				end
-			else
-				state.update_query(rv,'failed')
-			end
-			sleep wait if level < recursedepth
-		end
-		level += 1
-	end
-	state
-end
-
-def printresults(state,format,sep="\t")
-	case format
-	when 'text'
-		puts PassiveDNS::PDNSResult.members.join(sep)
-		puts state.to_s(sep)
-	when 'yaml'
-		puts state.to_yaml
-	when 'xml'
-		puts state.to_xml
-	when 'json'
-		puts state.to_json
-	when 'gdf'
-		puts state.to_gdf
-	when 'graphviz'
-		puts state.to_graphviz
-	when 'graphml'
-		puts state.to_graphml
-	end
-end
-
-def usage(letter_map)
-  databases = letter_map.keys.sort.join("")
-
-	puts "Usage: #{$0} [-d [#{databases}]] [-g|-v|-m|-c|-x|-y|-j|-t] [-os <sep>] [-f <file>] [-r#|-w#|-v] [-l <count>] <ip|domain|cidr>"
-	puts "  -d#{databases} uses all of the available passive dns databases"
-  letter_map.keys.sort.each do |l|
-    puts "  -d#{l} use #{letter_map[l][0]}"
-  end
-	puts "  -dvt uses VirusTotal and TCPIPUtils (for example)"
-	puts ""
-	puts "  -g outputs a link-nodal GDF visualization definition"
-	puts "  -v outputs a link-nodal graphviz visualization definition"
-	puts "  -m output a link-nodal graphml visualization definition"
-	puts "  -c outputs CSV"
-	puts "  -x outputs XML"
-	puts "  -y outputs YAML"
-	puts "  -j outputs JSON"
-	puts "  -t outputs ASCII text (default)"
-	puts "  -s <sep> specifies a field separator for text output, default is tab"
-	puts ""
-	puts "  -f[file] specifies a sqlite3 database used to read the current state - useful for large result sets and generating graphs of previous runs."
-	puts "  -r# specifies the levels of recursion to pull. **WARNING** This is quite taxing on the pDNS servers, so use judiciously (never more than 3 or so) or find yourself blocked!"
-	puts "  -w# specifies the amount of time to wait, in seconds, between queries (Default: 0)"
-	puts "  -v outputs debugging information"
-	puts "  -l <count> limits the number of records returned per passive dns database queried."
-	exit
-end
-
-opts = GetoptLong.new(
-	[ '--help', '-h', GetoptLong::NO_ARGUMENT ],
-	[ '--debug', '-z', GetoptLong::NO_ARGUMENT ],
-	[ '--database', '-d', GetoptLong::REQUIRED_ARGUMENT ],
-	
-	[ '--gdf', '-g', GetoptLong::NO_ARGUMENT ],
-	[ '--graphviz', '-v', GetoptLong::NO_ARGUMENT ],
-	[ '--graphml', '-m', GetoptLong::NO_ARGUMENT ],
-	[ '--csv', '-c', GetoptLong::NO_ARGUMENT ],
-	[ '--xml', '-x', GetoptLong::NO_ARGUMENT ],
-	[ '--yaml', '-y', GetoptLong::NO_ARGUMENT ],
-	[ '--json', '-j', GetoptLong::NO_ARGUMENT ],
-	[ '--text', '-t', GetoptLong::NO_ARGUMENT ],
-	[ '--sep', '-s', GetoptLong::REQUIRED_ARGUMENT ],
-	
-	[ '--sqlite3', '-f', GetoptLong::REQUIRED_ARGUMENT ],
-	[ '--recurse', '-r', GetoptLong::REQUIRED_ARGUMENT ],
-	[ '--wait', '-w', GetoptLong::REQUIRED_ARGUMENT ],
-	[ '--limit', '-l', GetoptLong::REQUIRED_ARGUMENT ]
-)
-
-letter_map = {}
-PassiveDNS.constants.each do |const|
-  if PassiveDNS.const_get(const).is_a?(Class) and PassiveDNS.const_get(const).superclass == PassiveDNS::PassiveDB
-    letter_map[PassiveDNS.const_get(const).option_letter] = [PassiveDNS.const_get(const).name, PassiveDNS.const_get(const).config_section_name]
-  end
-end
-
-# sets the default search methods
-pdnsdbs = []
-format = "text"
-sep = "\t"
-recursedepth = 1
-wait = 0
-res = nil
-debug = false
-sqlitedb = nil
-limit = nil
-
-opts.each do |opt, arg|
-	case opt
-	when '--help'
-		usage(letter_map)
-	when '--debug'
-		debug = true
-	when '--database'
-  	arg.split(//).each do |c|
-      if c == ','
-        next
-      elsif letter_map[c]
-        pdnsdbs << letter_map[c][1]
-      else
-        $stderr.puts "ERROR: Unknown passive DNS database identifier: #{c}."
-        usage(letter_map)
-      end
-    end
-	when '--gdf'
-		format = 'gdf'
-	when '--graphviz'
-		format = 'graphviz'
-	when '--graphml'
-		format = 'graphml'
-	when '--csv'
-		format = 'text'
-		sep = ','
-	when '--yaml'
-		format = 'yaml'
-	when '--xml'
-		format = 'xml'
-	when '--json'
-		format = 'json'
-	when '--text'
-		format = 'text'
-	when '--sep'
-		sep = arg
-	when '--recurse'
-		recursedepth = arg.to_i
-		if recursedepth > 3
-			$stderr.puts "WARNING:  a recursedepth of > 3 can be abusive, please reconsider: sleeping 60 seconds for sense to come to you (hint: hit CTRL-C)"
-			sleep 60
-		end
-	when '--wait'
-		wait = arg.to_i
-	when '--sqlite3'
-		sqlitedb = arg
-	when '--limit'
-		limit = arg.to_i
-	else
-		usage(letter_map)
-	end
-end
-
-if pdnsdbs.length == 0
-	pdnsdbs << "certee"
-end
-
-if pdnsdbs.index("bfk") and recursedepth > 1 and wait < 60
-	wait = 60
-	$stderr.puts "Enforcing a minimal 60 second wait when using BFK for recursive crawling"
-end
-
-if debug
-	$stderr.puts "Using the following databases: #{pdnsdbs.join(", ")}"
-	$stderr.puts "Recursions: #{recursedepth}, Wait time: #{wait}, Limit: #{limit or 'none'}"
-	if format == "text" or format == "csv"
-    	$stderr.puts "Output format: #{format} (sep=\"#{sep}\")"
-	else
-    	$stderr.puts "Output format: #{format}"
-	end
-	if ENV['http_proxy']
-		$stderr.puts "Using proxy settings: http_proxy=#{ENV['http_proxy']}, https_proxy=#{ENV['https_proxy']}"
-	end
-end
-
-state = nil
-if sqlitedb
-	state = PassiveDNS::PDNSToolStateDB.new(sqlitedb)
-else
-	state = PassiveDNS::PDNSToolState.new
-end
-state.debug = true if debug
-
-pdnsclient = PassiveDNS::Client.new(pdnsdbs)
-pdnsclient.debug = debug
-
-if ARGV.length > 0
-	ARGV.each do |arg|
-		state.add_query(arg,'pending',0)
-	end
-else
-	$stdin.each_line do |l|
-		state.add_query(l.chomp,'pending',0)
-	end
-end
-pdnslookup(state,pdnsclient,recursedepth,wait,debug,limit)
-printresults(state,format,sep)
+puts PassiveDNS::CLI.run(ARGV)

data/lib/passivedns/client.rb

@@ -3,15 +3,18 @@ require "passivedns/client/version"
 # This code is released under the LGPL: http://www.gnu.org/licenses/lgpl-3.0.txt
 # Please note that use of any passive dns database is subject to the terms of use of that passive dns database.  Use of this script in violation of their terms is not encouraged in any way.  Also, please do not add any obfuscation to try to work around their terms of service.  If you need special services, ask the providers for help/permission.
 # Remember, these passive DNS operators are my friends.  I don't want to have a row with them because some asshat used this library to abuse them.
-require 'passivedns/client/passivedb.rb'
-require 'passivedns/client/bfk.rb'
-require 'passivedns/client/certee.rb'
-require 'passivedns/client/dnsdb.rb'
-require 'passivedns/client/virustotal.rb'
-require 'passivedns/client/tcpiputils.rb'
-require 'passivedns/client/cn360.rb'
-require 'passivedns/client/state.rb'
-require 'passivedns/client/mnemonic.rb'
+require 'passivedns/client/state'
+require 'passivedns/client/passivedb'
+
+require 'passivedns/client/bfk'
+require 'passivedns/client/circl'
+require 'passivedns/client/cn360'
+require 'passivedns/client/dnsdb'
+require 'passivedns/client/mnemonic'
+require 'passivedns/client/passivetotal'
+require 'passivedns/client/tcpiputils'
+require 'passivedns/client/virustotal'
+
 require 'configparser'
 require 'pp'
 
@@ -20,7 +23,7 @@ module PassiveDNS
 	class PDNSResult < Struct.new(:source, :response_time, :query, :answer, :rrtype, :ttl, :firstseen, :lastseen, :count); end
 
 	class Client
-		def initialize(pdns=['bfk','certee','dnsdb','virustotal','tcpiputils','cn360','mnemonic'], configfile="#{ENV['HOME']}/.passivedns-client")
+		def initialize(pdns=['bfk','dnsdb','virustotal','tcpiputils','cn360','mnemonic','passivetotal','CIRCL'], configfile="#{ENV['HOME']}/.passivedns-client")
       cp = ConfigParser.new(configfile)
       # this creates a map of all the PassiveDNS provider names and their classes
       class_map = {}

data/lib/passivedns/client/circl.rb

@@ -0,0 +1,80 @@
+# DESCRIPTION: Module to query PassiveTotal's passive DNS repository
+
+require 'net/http'
+require 'net/https'
+require 'openssl'
+require 'pp'
+
+module PassiveDNS
+	class Circl < PassiveDB
+    # override
+    def self.name
+      "CIRCL"
+    end
+    #override
+    def self.config_section_name
+      "CIRCL"
+    end
+    #override
+    def self.option_letter
+      "c"
+    end
+    
+    attr_accessor :debug
+		def initialize(options={})
+      @debug = options[:debug] || true
+      @username = options["USERNAME"]
+      @password = options["PASSWORD"]
+      @auth_token = options["AUTH_TOKEN"]
+      @url = options["URL"] || "https://www.circl.lu/pdns/query"
+		end
+
+		def parse_json(page,query,response_time=0)
+ 			res = []
+			# need to remove the json_class tag or the parser will crap itself trying to find a class to align it to
+      page.split(/\n/).each do |line|
+        row = JSON.parse(line)
+				res << PDNSResult.new(self.class.name,response_time,
+          row['rrname'], row['rdata'], row['rrtype'], 0, 
+          row['time_first'], row['time_last'], row['count'])
+      end
+			res
+		rescue Exception => e
+			$stderr.puts "#{self.class.name} Exception: #{e}"
+			raise e
+		end
+
+		def lookup(label, limit=nil)
+			$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
+			Timeout::timeout(240) {
+				url = @url+"/"+label
+				$stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
+				url = URI.parse url
+				http = Net::HTTP.new(url.host, url.port)
+				http.use_ssl = (url.scheme == 'https')
+				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+				http.verify_depth = 5
+				request = Net::HTTP::Get.new(url.request_uri)
+				request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
+        if @username
+          request.basic_auth(@username, @password)
+        end
+        if @auth_token
+          request.add_field("Authorization", @auth_token)
+        end
+				t1 = Time.now
+				response = http.request(request)
+				t2 = Time.now
+				recs = parse_json(response.body, label, t2-t1)
+				if limit
+					recs[0,limit]
+				else
+					recs
+				end
+			}
+		rescue Timeout::Error => e
+			$stderr.puts "#{self.class.name} lookup timed out: #{label}"
+		end
+	end
+  CIRCL = PassiveDNS::Circl
+end

data/lib/passivedns/client/cli.rb

@@ -0,0 +1,245 @@
+require 'getoptlong'
+require 'structformatter'
+require 'getoptlong'
+require 'yaml'
+
+module PassiveDNS
+	class CLInterface
+    def self.parse_command_line(args)
+      opts = GetoptLong.new(
+      	[ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+      	[ '--debug', '-z', GetoptLong::NO_ARGUMENT ],
+      	[ '--database', '-d', GetoptLong::REQUIRED_ARGUMENT ],
+	
+      	[ '--gdf', '-g', GetoptLong::NO_ARGUMENT ],
+      	[ '--graphviz', '-v', GetoptLong::NO_ARGUMENT ],
+      	[ '--graphml', '-m', GetoptLong::NO_ARGUMENT ],
+      	[ '--csv', '-c', GetoptLong::NO_ARGUMENT ],
+      	[ '--xml', '-x', GetoptLong::NO_ARGUMENT ],
+      	[ '--yaml', '-y', GetoptLong::NO_ARGUMENT ],
+      	[ '--json', '-j', GetoptLong::NO_ARGUMENT ],
+      	[ '--text', '-t', GetoptLong::NO_ARGUMENT ],
+      	[ '--sep', '-s', GetoptLong::REQUIRED_ARGUMENT ],
+	
+      	[ '--sqlite3', '-f', GetoptLong::REQUIRED_ARGUMENT ],
+      	[ '--recurse', '-r', GetoptLong::REQUIRED_ARGUMENT ],
+      	[ '--wait', '-w', GetoptLong::REQUIRED_ARGUMENT ],
+      	[ '--limit', '-l', GetoptLong::REQUIRED_ARGUMENT ]
+      )
+
+      letter_map = {}
+      PassiveDNS.constants.each do |const|
+        if PassiveDNS.const_get(const).is_a?(Class) and PassiveDNS.const_get(const).superclass == PassiveDNS::PassiveDB
+          letter_map[PassiveDNS.const_get(const).option_letter] = [PassiveDNS.const_get(const).name, PassiveDNS.const_get(const).config_section_name]
+        end
+      end
+
+      # sets the default search methods
+      options = {
+        :pdnsdbs => [],
+        :format => "text",
+        :sep => "\t",
+        :recursedepth => 1,
+        :wait => 0,
+        :res => nil,
+        :debug => false,
+        :sqlitedb => nil,
+        :limit => nil
+      }
+
+      opts.each do |opt, arg|
+      	case opt
+      	when '--help'
+      		puts usage(letter_map)
+          exit
+      	when '--debug'
+      		options[:debug] = true
+      	when '--database'
+        	arg.split(//).each do |c|
+            if c == ','
+              next
+            elsif letter_map[c]
+              options[:pdnsdbs] << letter_map[c][1]
+            else
+              $stderr.puts "ERROR: Unknown passive DNS database identifier: #{c}."
+              usage(letter_map)
+            end
+          end
+      	when '--gdf'
+      		options[:format] = 'gdf'
+      	when '--graphviz'
+      		options[:format] = 'graphviz'
+      	when '--graphml'
+      		options[:format] = 'graphml'
+      	when '--csv'
+      		options[:format] = 'text'
+      		options[:sep] = ','
+      	when '--yaml'
+      		options[:format] = 'yaml'
+      	when '--xml'
+      		options[:format] = 'xml'
+      	when '--json'
+      		options[:format] = 'json'
+      	when '--text'
+      		options[:format] = 'text'
+      	when '--sep'
+      		options[:sep] = arg
+      	when '--recurse'
+      		options[:recursedepth] = arg.to_i
+      		if options[:recursedepth] > 3
+      			$stderr.puts "WARNING:  a recursedepth of > 3 can be abusive, please reconsider: sleeping 60 seconds for sense to come to you (hint: hit CTRL-C)"
+      			sleep 60
+      		end
+      	when '--wait'
+      		options[:wait] = arg.to_i
+      	when '--sqlite3'
+      		options[:sqlitedb] = arg
+      	when '--limit'
+      		otions[:limit] = arg.to_i
+      	else
+      		puts usage(letter_map)
+          exit
+      	end
+      end
+
+      if options[:pdnsdbs].length == 0
+      	options[:pdnsdbs] << "bfk"
+      end
+
+      if options[:pdnsdbs].index("bfk") and recursedepth > 1 and wait < 60
+      	options[:wait] = 60
+      	$stderr.puts "Enforcing a minimal 60 second wait when using BFK for recursive crawling"
+      end
+
+      if options[:debug]
+      	$stderr.puts "Using the following databases: #{options[:pdnsdbs].join(", ")}"
+      	$stderr.puts "Recursions: #{options[:recursedepth]}, Wait time: #{options[:wait]}, Limit: #{options[:limit] or 'none'}"
+      	if options[:format] == "text" or options[:format] == "csv"
+          	$stderr.puts "Output format: #{options[:format]} (sep=\"#{options[:sep]}\")"
+      	else
+          	$stderr.puts "Output format: #{options[:format]}"
+      	end
+      	if ENV['http_proxy']
+      		$stderr.puts "Using proxy settings: http_proxy=#{ENV['http_proxy']}, https_proxy=#{ENV['https_proxy']}"
+      	end
+      end
+      
+      [options, args]
+    end
+    
+    def self.usage(letter_map)
+      databases = letter_map.keys.sort.join("")
+      help_text = "\n"
+    	help_text << "Usage: #{$0} [-d [#{databases}]] [-g|-v|-m|-c|-x|-y|-j|-t] [-os <sep>] [-f <file>] [-r#|-w#|-v] [-l <count>] <ip|domain|cidr>\n"
+      help_text << "Passive DNS Providers"
+    	help_text << "  -d#{databases} uses all of the available passive dns database\n"
+      letter_map.keys.sort.each do |l|
+        help_text << "  -d#{l} use #{letter_map[l][0]}\n"
+      end
+    	help_text << "  -dvt uses VirusTotal and TCPIPUtils (for example)\n"
+    	help_text << "\n"
+      help_text << "Output Formatting\n"
+    	help_text << "  -g link-nodal GDF visualization definition\n"
+    	help_text << "  -v link-nodal graphviz visualization definition\n"
+    	help_text << "  -m link-nodal graphml visualization definition\n"
+    	help_text << "  -c CSV\n"
+    	help_text << "  -x XML\n"
+    	help_text << "  -y YAML\n"
+    	help_text << "  -j JSON\n"
+    	help_text << "  -t ASCII text (default)\n"
+    	help_text << "  -s <sep> specifies a field separator for text output, default is tab\n"
+    	help_text << "\n"
+      help_text << "State and Recusion\n"
+    	help_text << "  -f[file] specifies a sqlite3 database used to read the current state - useful for large result sets and generating graphs of previous runs.\n"
+    	help_text << "  -r# specifies the levels of recursion to pull. **WARNING** This is quite taxing on the pDNS servers, so use judiciously (never more than 3 or so) or find yourself blocked!\n"
+    	help_text << "  -w# specifies the amount of time to wait, in seconds, between queries (Default: 0)\n"
+    	help_text << "  -l <count> limits the number of records returned per passive dns database queried.\n"
+    	help_text << "\n"
+      help_text << "Getting Help\n"
+    	help_text << "  -v debugging information\n"
+      
+    	help_text
+    end
+    
+    def self.pdnslookup(state, pdnsclient, options)
+      recursedepth = options[:recursedepth]
+      wait = options[:wait]
+      debug = options[:debug]
+      limit = options[:limit]
+    	puts "pdnslookup: #{state.level} #{recursedepth}" if debug
+    	level = 0
+    	while level < recursedepth
+    		puts "pdnslookup: #{level} < #{recursedepth}" if debug
+    		state.each_query(recursedepth) do |q|
+    			rv = pdnsclient.query(q,limit)
+    			if rv
+    				rv.each do |r|
+    					if ["A","AAAA","NS","CNAME","PTR"].index(r.rrtype)
+    						puts "pdnslookup: #{r.to_s}" if debug
+    						state.add_result(r)
+    					end
+    				end
+    			else
+    				state.update_query(rv,'failed')
+    			end
+    			sleep wait if level < recursedepth
+    		end
+    		level += 1
+    	end
+    	state
+    end
+    
+    def self.results_to_s(state,options)
+      format = options[:format]
+      sep = options[:sep]
+    	case format
+    	when 'text'
+    		PassiveDNS::PDNSResult.members.join(sep)+"\n"+state.to_s(sep)
+    	when 'yaml'
+    		state.to_yaml
+    	when 'xml'
+    		state.to_xml
+    	when 'json'
+    		state.to_json
+    	when 'gdf'
+    		state.to_gdf
+    	when 'graphviz'
+    		state.to_graphviz
+    	when 'graphml'
+    		state.to_graphml
+    	end
+    end
+    
+    def self.create_state(sqlitedb=nil)
+      state = nil
+      if sqlitedb
+      	state = PassiveDNS::PDNSToolStateDB.new(sqlitedb)
+      else
+      	state = PassiveDNS::PDNSToolState.new
+      end
+    end
+    
+    def self.run(args)
+      options, items = parse_command_line(args)
+      state = create_state(options[:sqlitedb])
+      state.debug = options[:debug]
+
+      pdnsclient = PassiveDNS::Client.new(options[:pdnsdbs])
+      pdnsclient.debug = options[:debug]
+      
+      if ARGV.length > 0
+      	ARGV.each do |arg|
+      		state.add_query(arg,'pending',0)
+      	end
+      else
+      	$stdin.each_line do |l|
+      		state.add_query(l.chomp,'pending',0)
+      	end
+      end
+      pdnslookup(state,pdnsclient,options)
+      results_to_s(state,options)
+    end
+  end
+  CLI = PassiveDNS::CLInterface
+end
+

data/lib/passivedns/client/passivetotal.rb

@@ -0,0 +1,77 @@
+# DESCRIPTION: Module to query PassiveTotal's passive DNS repository
+
+require 'net/http'
+require 'net/https'
+require 'openssl'
+
+module PassiveDNS
+	class PassiveTotal < PassiveDB
+    # override
+    def self.name
+      "PassiveTotal"
+    end
+    #override
+    def self.config_section_name
+      "passivetotal"
+    end
+    #override
+    def self.option_letter
+      "p"
+    end
+    
+    attr_accessor :debug
+		def initialize(options={})
+      @debug = options[:debug] || false
+      @apikey = options["APIKEY"] || raise("#{self.class.name} requires an APIKEY")
+      @url = options["URL"] || "https://www.passivetotal.org/api/passive"
+		end
+
+		def parse_json(page,query,response_time=0)
+ 			res = []
+			# need to remove the json_class tag or the parser will crap itself trying to find a class to align it to
+			data = JSON.parse(page)
+			if data['results']
+        query = data['results']['value']
+				data['results']['resolutions'].each do |row|
+          first_seen = row['firstSeen']
+          last_seen = row['lastSeen']
+          value = row['value']
+          source = row['source'].join(",")
+					res << PDNSResult.new(self.class.name+"/"+source,response_time,
+            query, value, "A", 0, first_seen, last_seen)
+				end
+			end
+			res
+		rescue Exception => e
+			$stderr.puts "#{self.class.name} Exception: #{e}"
+			raise e
+		end
+
+		def lookup(label, limit=nil)
+			$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
+			Timeout::timeout(240) {
+				url = @url
+				$stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
+				url = URI.parse url
+				http = Net::HTTP.new(url.host, url.port)
+				http.use_ssl = (url.scheme == 'https')
+				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+				http.verify_depth = 5
+				request = Net::HTTP::Post.new(url.request_uri)
+				request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
+        request.set_form_data({"apikey" => @apikey, "value" => label})
+				t1 = Time.now
+				response = http.request(request)
+				t2 = Time.now
+				recs = parse_json(response.body, label, t2-t1)
+				if limit
+					recs[0,limit]
+				else
+					recs
+				end
+			}
+		rescue Timeout::Error => e
+			$stderr.puts "#{self.class.name} lookup timed out: #{label}"
+		end
+	end
+end

data/lib/passivedns/client/version.rb

@@ -1,5 +1,5 @@
 module PassiveDNS
   class Client
-    VERSION = "2.0.1"
+    VERSION = "2.0.2"
   end
 end

data/passivedns-client.gemspec

@@ -22,6 +22,7 @@ Gem::Specification.new do |spec|
 	spec.add_runtime_dependency 'sqlite3', '>= 1.3.3'
 	spec.add_runtime_dependency 'structformatter', '~> 0.0.1'
   spec.add_runtime_dependency 'configparser', '~> 0.1.3'
+  spec.add_development_dependency "minitest", "~> 5.5"
 	spec.add_development_dependency "bundler", "~> 1.3"
 	spec.add_development_dependency "rake"
 

data/test/helper.rb

@@ -1,2 +1,6 @@
-require 'test/unit'
+require 'minitest/autorun'
+require 'minitest/test'
+require 'minitest/unit'
+require 'minitest/pride'
+include MiniTest::Assertions
 require File.expand_path('../../lib/passivedns/client.rb', __FILE__)

data/test/test_passivedns-client.rb

@@ -9,7 +9,7 @@ end
 require_relative 'helper'
 require 'configparser'
 
-class TestPassiveDnsQuery < Test::Unit::TestCase
+class TestPassiveDnsQuery < Minitest::Test
   
   def setup
     configfile="#{ENV['HOME']}/.passivedns-client"
@@ -23,194 +23,227 @@ class TestPassiveDnsQuery < Test::Unit::TestCase
   end
     
 	def test_instantiate_Nonexisting_Client
-		assert_raise RuntimeError do
+		assert_raises RuntimeError do
 			PassiveDNS::Client.new(['doesnotexist'])
 		end
 	end
 
 	def test_instantiate_All_Clients
-		assert_nothing_raised do
-			PassiveDNS::Client.new()
-		end
+		PassiveDNS::Client.new()
 	end
 	
 	def test_instantiate_Passive_DNS_State
-		assert_not_nil(PassiveDNS::PDNSToolState.new)
+		refute_nil(PassiveDNS::PDNSToolState.new)
 	end
 	
 	def test_instantiate_Passive_DNS_State_database
 		if File.exists?("test/test.sqlite3")
 			File.unlink("test/test.sqlite3")
 		end
-		assert_not_nil(PassiveDNS::PDNSToolStateDB.new("test/test.sqlite3"))
+		refute_nil(PassiveDNS::PDNSToolStateDB.new("test/test.sqlite3"))
 		if File.exists?("test/test.sqlite3")
 			File.unlink("test/test.sqlite3")
 		end
 	end
 	
 	def test_BFK
-		assert_nothing_raised do
-			PassiveDNS::Client.new(['bfk'])
-		end
+    PassiveDNS::Client.new(['bfk'])
     d = PassiveDNS::BFK.new(@cp['bfk'] || {})
-    assert_not_nil(d)
+    refute_nil(d)
 		rows = d.lookup("example.org",3)
-		assert_not_nil(rows)
-		assert_not_nil(rows.to_s)
-		assert_not_nil(rows.to_xml)
-		assert_not_nil(rows.to_json)
-		assert_not_nil(rows.to_yaml)
+		refute_nil(rows)
+		refute_nil(rows.to_s)
+		refute_nil(rows.to_xml)
+		refute_nil(rows.to_json)
+		refute_nil(rows.to_yaml)
     assert_equal(3, rows.length)
     rows = d.lookup("8.8.8.8")
-    assert_not_nil(rows)
-    assert_not_nil(rows.to_s)
-    assert_not_nil(rows.to_xml)
-    assert_not_nil(rows.to_json)
-    assert_not_nil(rows.to_yaml)
+    refute_nil(rows)
+    refute_nil(rows.to_s)
+    refute_nil(rows.to_xml)
+    refute_nil(rows.to_json)
+    refute_nil(rows.to_yaml)
 	end
 	
 	def test_CERTEE
-    assert(false, "CERTEE is still offline")
-		assert_nothing_raised do
-			PassiveDNS::Client.new(['certee'])
-		end
+		PassiveDNS::Client.new(['certee'])
     d = PassiveDNS::CERTEE.new(@cp['certee'] || {})
-    assert_not_nil(d)
+    refute_nil(d)
 		rows = d.lookup("sim.cert.ee",3)
-		assert_not_nil(rows)
-		assert_not_nil(rows.to_s)
-		assert_not_nil(rows.to_xml)
-		assert_not_nil(rows.to_json)
-		assert_not_nil(rows.to_yaml)
+		refute_nil(rows)
+		refute_nil(rows.to_s)
+		refute_nil(rows.to_xml)
+		refute_nil(rows.to_json)
+		refute_nil(rows.to_yaml)
     assert_equal(3, rows.length)
     rows = d.lookup("8.8.8.8")
-    assert_not_nil(rows)
-    assert_not_nil(rows.to_s)
-    assert_not_nil(rows.to_xml)
-    assert_not_nil(rows.to_json)
-    assert_not_nil(rows.to_yaml)
+    refute_nil(rows)
+    refute_nil(rows.to_s)
+    refute_nil(rows.to_xml)
+    refute_nil(rows.to_json)
+    refute_nil(rows.to_yaml)
 	end
 
 	def test_DNSDB
-		assert_nothing_raised do
-			PassiveDNS::Client.new(['dnsdb'])
-		end
+		PassiveDNS::Client.new(['dnsdb'])
     d = PassiveDNS::DNSDB.new(@cp['dnsdb'] || {})
-    assert_not_nil(d)
+    refute_nil(d)
 		rows = d.lookup("example.org",3)
-		assert_not_nil(rows)
-		assert_not_nil(rows.to_s)
-		assert_not_nil(rows.to_xml)
-		assert_not_nil(rows.to_json)
-		assert_not_nil(rows.to_yaml)
+		refute_nil(rows)
+		refute_nil(rows.to_s)
+		refute_nil(rows.to_xml)
+		refute_nil(rows.to_json)
+		refute_nil(rows.to_yaml)
     assert_equal(3, rows.length) # this will fail since DNSDB has an off by one error
     rows = d.lookup("8.8.8.8")
-    assert_not_nil(rows)
-    assert_not_nil(rows.to_s)
-    assert_not_nil(rows.to_xml)
-    assert_not_nil(rows.to_json)
-    assert_not_nil(rows.to_yaml)
+    refute_nil(rows)
+    refute_nil(rows.to_s)
+    refute_nil(rows.to_xml)
+    refute_nil(rows.to_json)
+    refute_nil(rows.to_yaml)
 	end
 
 	def test_VirusTotal
-		assert_nothing_raised do
-			PassiveDNS::Client.new(['virustotal'])
-		end
+		PassiveDNS::Client.new(['virustotal'])
     d = PassiveDNS::VirusTotal.new(@cp['virustotal'] || {})
-    assert_not_nil(d)
+    refute_nil(d)
 		rows = d.lookup("google.com",3)
-		assert_not_nil(rows)
-		assert_not_nil(rows.to_s)
-		assert_not_nil(rows.to_xml)
-		assert_not_nil(rows.to_json)
-		assert_not_nil(rows.to_yaml)
+		refute_nil(rows)
+		refute_nil(rows.to_s)
+		refute_nil(rows.to_xml)
+		refute_nil(rows.to_json)
+		refute_nil(rows.to_yaml)
     assert_equal(3, rows.length)
     rows = d.lookup("8.8.8.8")
-    assert_not_nil(rows)
-    assert_not_nil(rows.to_s)
-    assert_not_nil(rows.to_xml)
-    assert_not_nil(rows.to_json)
-    assert_not_nil(rows.to_yaml)
+    refute_nil(rows)
+    refute_nil(rows.to_s)
+    refute_nil(rows.to_xml)
+    refute_nil(rows.to_json)
+    refute_nil(rows.to_yaml)
 	end
   
   def test_TCPIPUtils
-    assert_nothing_raised do
-      PassiveDNS::Client.new(['tcpiputils'])
-    end
+    PassiveDNS::Client.new(['tcpiputils'])
     d = PassiveDNS::TCPIPUtils.new(@cp['tcpiputils'] || {})
-    assert_not_nil(d)
+    refute_nil(d)
     rows = d.lookup("example.org")
-    assert_not_nil(rows)
-    assert_not_nil(rows.to_s)
-    assert_not_nil(rows.to_xml)
-    assert_not_nil(rows.to_json)
-    assert_not_nil(rows.to_yaml)
+    refute_nil(rows)
+    refute_nil(rows.to_s)
+    refute_nil(rows.to_xml)
+    refute_nil(rows.to_json)
+    refute_nil(rows.to_yaml)
     rows = d.lookup("example.org",3)
-    assert_not_nil(rows)
-    assert_not_nil(rows.to_s)
-    assert_not_nil(rows.to_xml)
-    assert_not_nil(rows.to_json)
-    assert_not_nil(rows.to_yaml)
+    refute_nil(rows)
+    refute_nil(rows.to_s)
+    refute_nil(rows.to_xml)
+    refute_nil(rows.to_json)
+    refute_nil(rows.to_yaml)
     assert_equal(3, rows.length)
     rows = d.lookup("8.8.8.8")
-    assert_not_nil(rows)
-    assert_not_nil(rows.to_s)
-    assert_not_nil(rows.to_xml)
-    assert_not_nil(rows.to_json)
-    assert_not_nil(rows.to_yaml)
+    refute_nil(rows)
+    refute_nil(rows.to_s)
+    refute_nil(rows.to_xml)
+    refute_nil(rows.to_json)
+    refute_nil(rows.to_yaml)
   end
 
   def test_cn360
-    assert_nothing_raised do
-      PassiveDNS::Client.new(['cn360'])
-    end
+    PassiveDNS::Client.new(['cn360'])
     d = PassiveDNS::CN360.new(@cp['cn360'] || {})
-    assert_not_nil(d)
+    refute_nil(d)
     rows = d.lookup("example.org")
-    assert_not_nil(rows)
-    assert_not_nil(rows.to_s)
-    assert_not_nil(rows.to_xml)
-    assert_not_nil(rows.to_json)
-    assert_not_nil(rows.to_yaml)
+    refute_nil(rows)
+    refute_nil(rows.to_s)
+    refute_nil(rows.to_xml)
+    refute_nil(rows.to_json)
+    refute_nil(rows.to_yaml)
     rows = d.lookup("example.org",3)
-    assert_not_nil(rows)
-    assert_not_nil(rows.to_s)
-    assert_not_nil(rows.to_xml)
-    assert_not_nil(rows.to_json)
-    assert_not_nil(rows.to_yaml)
+    refute_nil(rows)
+    refute_nil(rows.to_s)
+    refute_nil(rows.to_xml)
+    refute_nil(rows.to_json)
+    refute_nil(rows.to_yaml)
     assert_equal(3, rows.length)
     rows = d.lookup("8.8.8.8")
-    assert_not_nil(rows)
-    assert_not_nil(rows.to_s)
-    assert_not_nil(rows.to_xml)
-    assert_not_nil(rows.to_json)
-    assert_not_nil(rows.to_yaml)
+    refute_nil(rows)
+    refute_nil(rows.to_s)
+    refute_nil(rows.to_xml)
+    refute_nil(rows.to_json)
+    refute_nil(rows.to_yaml)
 	end
   
   def test_nmemonic
-		assert_nothing_raised do
-			PassiveDNS::Client.new(['mnemonic'])
-		end
+		PassiveDNS::Client.new(['mnemonic'])
     d = PassiveDNS::Mnemonic.new(@cp['mnemonic'] || {})
-    assert_not_nil(d)
+    refute_nil(d)
+		rows = d.lookup("example.org")
+		refute_nil(rows)
+		refute_nil(rows.to_s)
+		refute_nil(rows.to_xml)
+		refute_nil(rows.to_json)
+		refute_nil(rows.to_yaml)
+		rows = d.lookup("example.org",3)
+		refute_nil(rows)
+		refute_nil(rows.to_s)
+		refute_nil(rows.to_xml)
+		refute_nil(rows.to_json)
+		refute_nil(rows.to_yaml)
+		assert_equal(3, rows.length)
+		rows = d.lookup("8.8.8.8")
+		refute_nil(rows)
+		refute_nil(rows.to_s)
+		refute_nil(rows.to_xml)
+		refute_nil(rows.to_json)
+		refute_nil(rows.to_yaml)
+	end
+  
+  def test_passivetotal
+		PassiveDNS::Client.new(['passivetotal'])
+    d = PassiveDNS::PassiveTotal.new(@cp['passivetotal'] || {})
+    refute_nil(d)
+		rows = d.lookup("example.org")
+		refute_nil(rows)
+		refute_nil(rows.to_s)
+		refute_nil(rows.to_xml)
+		refute_nil(rows.to_json)
+		refute_nil(rows.to_yaml)
+		rows = d.lookup("example.org",3)
+		refute_nil(rows)
+		refute_nil(rows.to_s)
+		refute_nil(rows.to_xml)
+		refute_nil(rows.to_json)
+		refute_nil(rows.to_yaml)
+		assert_equal(3, rows.length)
+		rows = d.lookup("8.8.8.8")
+		refute_nil(rows)
+		refute_nil(rows.to_s)
+		refute_nil(rows.to_xml)
+		refute_nil(rows.to_json)
+		refute_nil(rows.to_yaml)
+	end
+    
+  def test_circl
+		PassiveDNS::Client.new(['CIRCL'])
+    d = PassiveDNS::CIRCL.new(@cp['CIRCL'] || {})
+    refute_nil(d)
 		rows = d.lookup("example.org")
-		assert_not_nil(rows)
-		assert_not_nil(rows.to_s)
-		assert_not_nil(rows.to_xml)
-		assert_not_nil(rows.to_json)
-		assert_not_nil(rows.to_yaml)
+		refute_nil(rows)
+		refute_nil(rows.to_s)
+		refute_nil(rows.to_xml)
+		refute_nil(rows.to_json)
+		refute_nil(rows.to_yaml)
 		rows = d.lookup("example.org",3)
-		assert_not_nil(rows)
-		assert_not_nil(rows.to_s)
-		assert_not_nil(rows.to_xml)
-		assert_not_nil(rows.to_json)
-		assert_not_nil(rows.to_yaml)
+		refute_nil(rows)
+		refute_nil(rows.to_s)
+		refute_nil(rows.to_xml)
+		refute_nil(rows.to_json)
+		refute_nil(rows.to_yaml)
 		assert_equal(3, rows.length)
 		rows = d.lookup("8.8.8.8")
-		assert_not_nil(rows)
-		assert_not_nil(rows.to_s)
-		assert_not_nil(rows.to_xml)
-		assert_not_nil(rows.to_json)
-		assert_not_nil(rows.to_yaml)
+		refute_nil(rows)
+		refute_nil(rows.to_s)
+		refute_nil(rows.to_xml)
+		refute_nil(rows.to_json)
+		refute_nil(rows.to_yaml)
 	end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: passivedns-client
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.0.2
 platform: ruby
 authors:
 - chrislee35
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-20 00:00:00.000000000 Z
+date: 2015-04-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.3
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.5'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -112,11 +126,13 @@ files:
 - bin/pdnstool
 - lib/passivedns/client.rb
 - lib/passivedns/client/bfk.rb
-- lib/passivedns/client/certee.rb
+- lib/passivedns/client/circl.rb
+- lib/passivedns/client/cli.rb
 - lib/passivedns/client/cn360.rb
 - lib/passivedns/client/dnsdb.rb
 - lib/passivedns/client/mnemonic.rb
 - lib/passivedns/client/passivedb.rb
+- lib/passivedns/client/passivetotal.rb
 - lib/passivedns/client/state.rb
 - lib/passivedns/client/tcpiputils.rb
 - lib/passivedns/client/version.rb

data/lib/passivedns/client/certee.rb

@@ -1,62 +0,0 @@
-require 'socket'
-require_relative 'passivedb'
-
-module PassiveDNS
-	class CERTEE < PassiveDB
-    # override
-    def self.name
-      "CERTEE"
-    end
-    #override
-    def self.config_section_name
-      "certee"
-    end
-    #override
-    def self.option_letter
-      "e"
-    end
-    
-    attr_accessor :debug
-		def initialize(options={})
-      @debug = options[:debug] || false
-      @host = options["HOST"] || "sim.cert.ee"
-      @port = options["PORT"].to_i || 43
-		end
-
-    # override
-		def lookup(label, limit=nil)
-			$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
-			recs = []
-			begin
-				t1 = Time.now
-				s = TCPSocket.new(@host,@port)
-				s.puts(label)
-				s.each_line do |l|
-          if l =~ /Traceback \(most recent call last\):/
-            # there is a bug in the CERTEE lookup tool
-            raise "#{self.class.name} is currently offline"
-          end
-					(lbl,ans,fs,ls) = l.chomp.split(/\t/)
-					rrtype = 'A'
-					if ans =~ /^\d+\.\d+\.\d+\.\d+$/
-						rrtype = 'A'
-					elsif ans =~ /^ns/
-						rrtype = 'NS'
-					else
-						rrtype = 'CNAME'
-					end
-					t2 = Time.now
-					recs << PDNSResult.new(self.class.name,t2-t1,lbl,ans,rrtype,0,Time.parse(fs).utc.strftime("%Y-%m-%dT%H:%M:%SZ"),Time.parse(ls).utc.strftime("%Y-%m-%dT%H:%M:%SZ"))
-				end
-			rescue SocketError => e
-				$stderr.puts e
-			end
-			return nil unless recs.length > 0
-      if limit
-        recs = recs[0,limit]
-      else
-        recs
-      end
-		end
-	end
-end