passivedns-client 1.1.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 06d7ff1c185b1d52a22829c78ac3f393527f4d61
-  data.tar.gz: b6fb23b393f3d87948da754ce6c4a69476fb280f
+  metadata.gz: 41a0a59329a330d5830052e03a99a39f9ce5a819
+  data.tar.gz: 361993fddb9ef65fef2f4721dba03257b818dddd
 SHA512:
-  metadata.gz: ade09d0cbf82ea14e6ea216b228e9b1434d5cac2cfc3ab54145ca7be77142880f0fedc220e77839422f9e38976752fbdf6ffdf68fd3ecfde61792ea91d8bc2cd
-  data.tar.gz: 08d846c8d67746c2864b627346641079c8daba8efe151334ff284b5bc7d2922deac58628d294104571f32481e67759903d13cdc8e1f916b050625c4f8bd9b94c
+  metadata.gz: edbb6d872501226acc1aa067a432b63908f9063657f708d1ccf234cd207c681b85eb071c405a7fa772ac51acda85cb6cc71db2d18109282856dd91e19c87061a
+  data.tar.gz: 56c04abc0c89251b2303721a2ded1dee6da155bd4a66c43bb12de5ab52e3884fe32b21ec463c083d0e7900ed862068f31c343d58c3bd51d9142e855c908178fb

data/bin/pdnstool

@@ -4,198 +4,211 @@ require 'structformatter'
 require 'getoptlong'
 require 'yaml'
 
-#if __FILE__ == $0
-	def pdnslookup(state,pdnsclient,recursedepth=1,wait=0,debug=false)
-		puts "pdnslookup: #{state.level} #{recursedepth}" if debug
-		level = 0
-		while level < recursedepth
-			puts "pdnslookup: #{level} < #{recursedepth}" if debug
-			state.each_query(recursedepth) do |q|
-				rv = pdnsclient.query(q)
-				if rv
-					rv.each do |r|
-						if ["A","AAAA","NS","CNAME","PTR"].index(r.rrtype)
-							puts "pdnslookup: #{r.to_s}" if debug
-							state.add_result(r)
-						end
+def pdnslookup(state,pdnsclient,recursedepth=1,wait=0,debug=false,limit=nil)
+	puts "pdnslookup: #{state.level} #{recursedepth}" if debug
+	level = 0
+	while level < recursedepth
+		puts "pdnslookup: #{level} < #{recursedepth}" if debug
+		state.each_query(recursedepth) do |q|
+			rv = pdnsclient.query(q,limit)
+			if rv
+				rv.each do |r|
+					if ["A","AAAA","NS","CNAME","PTR"].index(r.rrtype)
+						puts "pdnslookup: #{r.to_s}" if debug
+						state.add_result(r)
 					end
-				else
-					state.update_query(rv,'failed')
 				end
-				sleep wait if level < recursedepth
+			else
+				state.update_query(rv,'failed')
 			end
-			level += 1
-		end
-		state
-	end
-	
-	def printresults(state,format,sep="\t")
-		case format
-		when 'text'
-			puts PassiveDNS::PDNSResult.members.join(sep)
-			puts state.to_s(sep)
-		when 'yaml'
-			puts state.to_yaml
-		when 'xml'
-			puts state.to_xml
-		when 'json'
-			puts state.to_json
-		when 'gdf'
-			puts state.to_gdf
-		when 'graphviz'
-			puts state.to_graphviz
-		when 'graphml'
-			puts state.to_graphml
+			sleep wait if level < recursedepth
 		end
+		level += 1
 	end
+	state
+end
 
-	def usage
-		puts "Usage: #{$0} [-a|-b|-e|-d|-i|-V] [-c|-x|-y|-j|-t] [-s <sep>] [-f <file>] [-r#|-w#|-l] <ip|domain|cidr>"
-		puts "  -a uses all of the available passive dns databases"
-		puts "  -b only use BFK"
-		puts "  -e only use CERT-EE"
-		puts "  -d only use DNSParse (default)"
-		puts "  -i only use DNSDB (formerly ISC)"
-		puts "  -V only use VirusTotal"
-		puts ""
-		puts "  -g outputs a link-nodal GDF visualization definition"
-		puts "  -v outputs a link-nodal graphviz visualization definition"
-		puts "  -m output a link-nodal graphml visualization definition"
-		puts "  -c outputs CSV"
-		puts "  -x outputs XML"
-		puts "  -y outputs YAML"
-		puts "  -j outputs JSON"
-		puts "  -t outputs ASCII text (default)"
-		puts "  -s <sep> specifies a field separator for text output, default is tab"
-		puts ""
-		puts "  -f[file] specifies a sqlite3 database used to read the current state - useful for large result sets and generating graphs of previous runs."
-		puts "  -r# specifies the levels of recursion to pull. **WARNING** This is quite taxing on the pDNS servers, so use judiciously (never more than 3 or so) or find yourself blocked!"
-		puts "  -w# specifies the amount of time to wait, in seconds, between queries (Default: 0)"
-		puts "  -l outputs debugging information"
-		exit
+def printresults(state,format,sep="\t")
+	case format
+	when 'text'
+		puts PassiveDNS::PDNSResult.members.join(sep)
+		puts state.to_s(sep)
+	when 'yaml'
+		puts state.to_yaml
+	when 'xml'
+		puts state.to_xml
+	when 'json'
+		puts state.to_json
+	when 'gdf'
+		puts state.to_gdf
+	when 'graphviz'
+		puts state.to_graphviz
+	when 'graphml'
+		puts state.to_graphml
 	end
+end
+
+def usage
+	puts "Usage: #{$0} [-d [bedvt]] [-og|-ov|-om|-oc|-ox|-oy|-oj|-ot] [-os <sep>] [-f <file>] [-r#|-w#|-v] [-l <count>] <ip|domain|cidr>"
+	puts "  -dbedvt uses all of the available passive dns databases"
+	puts "  -db only use BFK"
+	puts "  -de only use CERT-EE (default)"
+	puts "  -dd only use DNSDB (formerly ISC)"
+	puts "  -dv only use VirusTotal"
+  puts "  -dt only use TCPIPUtils"
+  puts "  -dvt uses VirusTotal and TCPIPUtils (for example)"
+	puts ""
+	puts "  -og outputs a link-nodal GDF visualization definition"
+	puts "  -ov outputs a link-nodal graphviz visualization definition"
+	puts "  -om output a link-nodal graphml visualization definition"
+	puts "  -oc outputs CSV"
+	puts "  -ox outputs XML"
+	puts "  -oy outputs YAML"
+	puts "  -oj outputs JSON"
+	puts "  -ot outputs ASCII text (default)"
+	puts "  -os <sep> specifies a field separator for text output, default is tab"
+	puts ""
+	puts "  -f[file] specifies a sqlite3 database used to read the current state - useful for large result sets and generating graphs of previous runs."
+	puts "  -r# specifies the levels of recursion to pull. **WARNING** This is quite taxing on the pDNS servers, so use judiciously (never more than 3 or so) or find yourself blocked!"
+	puts "  -w# specifies the amount of time to wait, in seconds, between queries (Default: 0)"
+	puts "  -v outputs debugging information"
+  puts "  -l <count> limits the number of records returned per passive dns database queried."
+	exit
+end
+
+opts = GetoptLong.new(
+	[ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+	[ '--debug', '-z', GetoptLong::NO_ARGUMENT ],
+  [ '--database', '-d', GetoptLong::REQUIRED_ARGUMENT ],
 	
-	opts = GetoptLong.new(
-		[ '--help', '-h', GetoptLong::NO_ARGUMENT ],
-		[ '--debug', '-l', GetoptLong::NO_ARGUMENT ],
-		[ '--all', '-a', GetoptLong::NO_ARGUMENT ],
-		[ '--bfk', '-b', GetoptLong::NO_ARGUMENT ],
-		[ '--dnsparse', '-d', GetoptLong::NO_ARGUMENT ],
-		[ '--certee', '-e', GetoptLong::NO_ARGUMENT ],
-		[ '--isc', '-i', GetoptLong::NO_ARGUMENT ],
-		[ '--virustotal', '-V', GetoptLong::NO_ARGUMENT ],
-		
-		[ '--gdf', '-g', GetoptLong::NO_ARGUMENT ],
-		[ '--graphviz', '-v', GetoptLong::NO_ARGUMENT ],
-		[ '--graphml', '-m', GetoptLong::NO_ARGUMENT ],
-		[ '--csv', '-c', GetoptLong::NO_ARGUMENT ],
-		[ '--xml', '-x', GetoptLong::NO_ARGUMENT ],
-		[ '--yaml', '-y', GetoptLong::NO_ARGUMENT ],
-		[ '--json', '-j', GetoptLong::NO_ARGUMENT ],
-		[ '--text', '-t', GetoptLong::NO_ARGUMENT ],
-		[ '--sep', '-s', GetoptLong::REQUIRED_ARGUMENT ],
-		
-		[ '--sqlite3', '-f', GetoptLong::REQUIRED_ARGUMENT ],
-		[ '--recurse', '-r', GetoptLong::REQUIRED_ARGUMENT ],
-		[ '--wait', '-w', GetoptLong::REQUIRED_ARGUMENT ]
-	)
-	
-	# sets the default search methods
-	pdnsdbs = []
-	format = "text"
-	sep = "\t"
-	recursedepth = 1
-	wait = 0
-	res = nil
-	debug = false
-	sqlitedb = nil
+	[ '--gdf', '-g', GetoptLong::NO_ARGUMENT ],
+	[ '--graphviz', '-v', GetoptLong::NO_ARGUMENT ],
+	[ '--graphml', '-m', GetoptLong::NO_ARGUMENT ],
+	[ '--csv', '-c', GetoptLong::NO_ARGUMENT ],
+	[ '--xml', '-x', GetoptLong::NO_ARGUMENT ],
+	[ '--yaml', '-y', GetoptLong::NO_ARGUMENT ],
+	[ '--json', '-j', GetoptLong::NO_ARGUMENT ],
+	[ '--text', '-t', GetoptLong::NO_ARGUMENT ],
+	[ '--sep', '-s', GetoptLong::REQUIRED_ARGUMENT ],
 	
-	opts.each do |opt, arg|
-		case opt
-		when '--help'
-			usage
-		when '--debug'
-			debug = true
-			$stderr.puts "Using proxy settings: http_proxy=#{ENV['http_proxy']}, https_proxy=#{ENV['https_proxy']}"
-		when '--all'
-			pdnsdbs << "bfk"
-			pdnsdbs << "certee"
-			pdnsdbs << "dnsparse"
-			pdnsdbs << "dnsdb"
-			pdnsdbs << "virustotal"
-		when '--bfk'
-			pdnsdbs << "bfk"
-		when '--certee'
-			pdnsdbs << "certee"
-		when '--dnsparse'
-			pdnsdbs << "dnsparse"
-		when '--isc'
-			pdnsdbs << "dnsdb"
-		when '--virustotal'
-			pdnsdbs << "virustotal"
-		when '--gdf'
-			format = 'gdf'
-		when '--graphviz'
-			format = 'graphviz'
-		when '--graphml'
-			format = 'graphml'
-		when '--csv'
-			format = 'text'
-			sep = ','
-		when '--yaml'
-			format = 'yaml'
-		when '--xml'
-			format = 'xml'
-		when '--json'
-			format = 'json'
-		when '--text'
-			format = 'text'
-		when '--sep'
-			sep = arg
-		when '--recurse'
-			recursedepth = arg.to_i
-			if recursedepth > 3
-				$stderr.puts "WARNING:  a recursedepth of > 3 can be abusive, please reconsider: sleeping 60 seconds for sense to come to you (hint: hit CTRL-C)"
-				sleep 60
-			end
-		when '--wait'
-			wait = arg.to_i
-		when '--sqlite3'
-			sqlitedb = arg
-		else
-			usage
+	[ '--sqlite3', '-f', GetoptLong::REQUIRED_ARGUMENT ],
+	[ '--recurse', '-r', GetoptLong::REQUIRED_ARGUMENT ],
+	[ '--wait', '-w', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--limit', '-l', GetoptLong::REQUIRED_ARGUMENT ]
+)
+
+# sets the default search methods
+pdnsdbs = []
+format = "text"
+sep = "\t"
+recursedepth = 1
+wait = 0
+res = nil
+debug = false
+sqlitedb = nil
+limit = nil
+
+opts.each do |opt, arg|
+	case opt
+	when '--help'
+		usage
+	when '--debug'
+		debug = true
+  when '--database'
+    arg.split(//).each do |c|
+      case c
+      when ','
+      when 'b'
+    		pdnsdbs << "bfk"
+      when 'e'
+    		pdnsdbs << "certee"
+      when 'd'
+    		pdnsdbs << "dnsdb"
+      when 'v'
+    		pdnsdbs << "virustotal"
+      when 't'
+        pdnsdbs << "tcpiputils"
+      else
+        raise "Unknown passive DNS database identifier: #{c}"
+      end
+    end
+	when '--gdf'
+		format = 'gdf'
+	when '--graphviz'
+		format = 'graphviz'
+	when '--graphml'
+		format = 'graphml'
+	when '--csv'
+		format = 'text'
+		sep = ','
+	when '--yaml'
+		format = 'yaml'
+	when '--xml'
+		format = 'xml'
+	when '--json'
+		format = 'json'
+	when '--text'
+		format = 'text'
+	when '--sep'
+		sep = arg
+	when '--recurse'
+		recursedepth = arg.to_i
+		if recursedepth > 3
+			$stderr.puts "WARNING:  a recursedepth of > 3 can be abusive, please reconsider: sleeping 60 seconds for sense to come to you (hint: hit CTRL-C)"
+			sleep 60
 		end
-	end
-	
-	if pdnsdbs.length == 0
-		pdnsdbs << "dnsparse"
-	end
-	
-	if pdnsdbs.index("bfk") and recursedepth > 1 and wait < 60
-		wait = 60
-		$stderr.puts "Enforcing a minimal 60 second wait when using BFK for recursive crawling"
-	end
-		
-	state = nil
-	if sqlitedb
-		state = PassiveDNS::PDNSToolStateDB.new(sqlitedb)
+	when '--wait'
+		wait = arg.to_i
+	when '--sqlite3'
+		sqlitedb = arg
+  when '--limit'
+    limit = arg.to_i
 	else
-		state = PassiveDNS::PDNSToolState.new
+		usage
 	end
-	state.debug = true if debug
-	
-	pdnsclient = PassiveDNS::Client.new(pdnsdbs)
-	pdnsclient.debug = debug
-	
-	if ARGV.length > 0
-		ARGV.each do |arg|
-			state.add_query(arg,'pending',0)
-		end
-	else
-		$stdin.each_line do |l|
-			state.add_query(l.chomp,'pending',0)
-		end
+end
+
+if pdnsdbs.length == 0
+	pdnsdbs << "certee"
+end
+
+if pdnsdbs.index("bfk") and recursedepth > 1 and wait < 60
+	wait = 60
+	$stderr.puts "Enforcing a minimal 60 second wait when using BFK for recursive crawling"
+end
+
+if debug
+	$stderr.puts "Using the following databases: #{pdnsdbs.join(", ")}"
+  $stderr.puts "Recursions: #{recursedepth}, Wait time: #{wait}, Limit: #{limit or 'none'}"
+  if format == "text" or format == "csv"
+    $stderr.puts "Output format: #{format} (sep=\"#{sep}\")"
+  else
+    $stderr.puts "Output format: #{format}"
+  end
+  if ENV['http_proxy']
+    $stderr.puts "Using proxy settings: http_proxy=#{ENV['http_proxy']}, https_proxy=#{ENV['https_proxy']}"
+  end
+end
+
+state = nil
+if sqlitedb
+	state = PassiveDNS::PDNSToolStateDB.new(sqlitedb)
+else
+	state = PassiveDNS::PDNSToolState.new
+end
+state.debug = true if debug
+
+pdnsclient = PassiveDNS::Client.new(pdnsdbs)
+pdnsclient.debug = debug
+
+if ARGV.length > 0
+	ARGV.each do |arg|
+		state.add_query(arg,'pending',0)
+	end
+else
+	$stdin.each_line do |l|
+		state.add_query(l.chomp,'pending',0)
 	end
-	pdnslookup(state,pdnsclient,recursedepth,wait,debug)
-	printresults(state,format,sep)
-#end
+end
+pdnslookup(state,pdnsclient,recursedepth,wait,debug,limit)
+printresults(state,format,sep)

data/lib/passivedns/client/bfk.rb

@@ -38,7 +38,7 @@ module PassiveDNS
 			raise e
 		end
 
-		def lookup(label)	
+		def lookup(label, limit=nil)	
 			$stderr.puts "DEBUG: BFKClient.lookup(#{label})" if @debug
 			Timeout::timeout(240) {
 				t1 = Time.now
@@ -48,7 +48,12 @@ module PassiveDNS
 					:http_basic_authentication => [@user,@pass]
 				) do |f|
 					t2 = Time.now
-					parse(f.read,t2-t1)
+					recs = parse(f.read,t2-t1)
+          if limit
+            recs[0,limit]
+          else
+            recs
+          end
 				end
 			}
 		rescue Timeout::Error => e

data/lib/passivedns/client/certee.rb

@@ -6,7 +6,7 @@ module PassiveDNS
 		attr_accessor :debug
 		def initialize
 		end
-		def lookup(label)
+		def lookup(label, limit=nil)
 			$stderr.puts "DEBUG: CERTEE.lookup(#{label})" if @debug
 			recs = []
 			begin
@@ -14,6 +14,10 @@ module PassiveDNS
 				s = TCPSocket.new(@@host,43)
 				s.puts(label)
 				s.each_line do |l|
+          if l =~ /Traceback \(most recent call last\):/
+            # there is a bug in the CERTEE lookup tool
+            raise "CERTEE is currently offline"
+          end
 					(lbl,ans,fs,ls) = l.chomp.split(/\t/)
 					rrtype = 'A'
 					if ans =~ /^\d+\.\d+\.\d+\.\d+$/
@@ -30,7 +34,11 @@ module PassiveDNS
 				$stderr.puts e
 			end
 			return nil unless recs.length > 0
-			recs
-		end	
+      if limit
+        recs = recs[0,limit]
+      else
+        recs
+      end
+		end
 	end
 end

data/lib/passivedns/client/dnsdb.rb

@@ -48,7 +48,7 @@ module PassiveDNS
 			raise e
 		end
 
-		def lookup(label)
+		def lookup(label, limit=nil)
 			$stderr.puts "DEBUG: DNSDB.lookup(#{label})" if @debug
 			Timeout::timeout(240) {
 				url = nil
@@ -63,14 +63,18 @@ module PassiveDNS
 				http.use_ssl = (url.scheme == 'https')
 				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
 				http.verify_depth = 5
-				request = Net::HTTP::Get.new(url.path)
+        path = url.path
+        if limit
+          path << "?limit=#{limit}"
+        end
+				request = Net::HTTP::Get.new(path)
 				request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
 				request.add_field("X-API-Key", @key)
 				request.add_field("Accept", "application/json")
 				t1 = Time.now
 				response = http.request(request)
 				t2 = Time.now
-				$stderr.puts response if @debug
+				$stderr.puts response.body if @debug
 				parse_json(response.body,t2-t1)
 			}
 		rescue Timeout::Error => e

data/lib/passivedns/client/tcpiputils.rb

@@ -0,0 +1,80 @@
+require 'net/http'
+require 'net/https'
+require 'openssl'
+require 'json'
+
+# Please read http://www.tcpiputils.com/terms-of-service under automated requests
+
+module PassiveDNS
+  class TCPIPUtils
+    attr_accessor :debug
+    def initialize(config="#{ENV['HOME']}/.tcpiputils")
+      @debug = false
+			if File.exist?(config)
+				@apikey = File.open(config).read.split(/\n/)[0]
+				$stderr.puts "DEBUG: TCPIPUtils#initialize(#{@apikey})" if @debug
+			else
+				raise "Error: Configuration file for TCPIPUtils is required for intialization
+Format of configuration file (default: #{ENV['HOME']}/.tcpiputils) is the 64 hex character apikey on one line.
+To obtain an API Key, go to http://www.tcpiputils.com/premium-access and purchase premium API access."
+			end
+    end
+    
+    def format_recs(reply_data, question, delta)
+      recs = []
+      reply_data.each do |key, data|
+        case key
+        when "ipv4"
+          data.each do |rec|
+            recs << PDNSResult.new("tcpiputils", delta, question, rec["ip"], "A", nil, nil, rec["updatedate"], nil)
+          end
+        when "ipv6"
+          data.each do |rec|
+            recs << PDNSResult.new("tcpiputils", delta, question, rec["ip"], "AAAA", nil, nil, rec["updatedate"], nil)
+          end
+        when "dns"
+          data.each do |rec|
+            recs << PDNSResult.new("tcpiputils", delta, question, rec["dns"], "NS", nil, nil, rec["updatedate"], nil)
+          end
+        when "mx"
+          data.each do |rec|
+            recs << PDNSResult.new("tcpiputils", delta, question, rec["dns"], "MX", nil, nil, rec["updatedate"], nil)
+          end
+        end
+      end
+      recs
+    end
+
+    def lookup(label, limit=nil)
+      $stderr.puts "DEBUG: TCPIPUtils.lookup(#{label})" if @debug
+      url = "https://www.utlsapi.com/api.php?version=1.0&apikey=#{@apikey}&type=domainipdnshistory&q=#{label}"
+      recs = []
+      Timeout::timeout(240) {
+				url = URI.parse url
+				http = Net::HTTP.new(url.host, url.port)
+				http.use_ssl = (url.scheme == 'https')
+				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+				http.verify_depth = 5
+				request = Net::HTTP::Get.new(url.path+"?"+url.query)
+				request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
+				t1 = Time.now
+				response = http.request(request)
+				delta = (Time.now - t1).to_f
+				reply = JSON.parse(response.body)
+        if reply["status"] and reply["status"] == "succeed"
+          question = reply["data"]["question"]
+          recs = format_recs(reply["data"], question, delta)
+        elsif reply["status"] and reply["status"] == "error"
+          raise "TCPIPUtils: error from web API: #{reply["data"]}"
+        end
+        if limit
+          recs[0,limit]
+        else
+          recs
+        end
+			}
+		rescue Timeout::Error => e
+			$stderr.puts "TCPIPUtils lookup timed out: #{label}"
+		end
+  end
+end

data/lib/passivedns/client/version.rb

@@ -1,5 +1,5 @@
 module PassiveDNS
   class Client
-    VERSION = "1.1.1"
+    VERSION = "1.3.0"
   end
 end

data/lib/passivedns/client/virustotal.rb

@@ -34,7 +34,7 @@ module PassiveDNS
 			raise e
 		end
 
-		def lookup(label)
+		def lookup(label, limit=nil)
 			$stderr.puts "DEBUG: VirusTotal.lookup(#{label})" if @debug
 			Timeout::timeout(240) {
 				url = nil
@@ -54,7 +54,12 @@ module PassiveDNS
 				t1 = Time.now
 				response = http.request(request)
 				t2 = Time.now
-				parse_json(response.body, label, t2-t1)
+				recs = parse_json(response.body, label, t2-t1)
+        if limit
+          recs[0,limit]
+        else
+          recs
+        end
 			}
 		rescue Timeout::Error => e
 			$stderr.puts "VirusTotal lookup timed out: #{label}"

data/lib/passivedns/client.rb

@@ -5,9 +5,9 @@ require "passivedns/client/version"
 # Remember, these passive DNS operators are my friends.  I don't want to have a row with them because some asshat used this library to abuse them.
 require 'passivedns/client/bfk.rb'
 require 'passivedns/client/certee.rb'
-require 'passivedns/client/dnsparse.rb'
 require 'passivedns/client/dnsdb.rb'
 require 'passivedns/client/virustotal.rb'
+require 'passivedns/client/tcpiputils.rb'
 require 'passivedns/client/state.rb'
 
 module PassiveDNS
@@ -15,7 +15,7 @@ module PassiveDNS
 	class PDNSResult < Struct.new(:source, :response_time, :query, :answer, :rrtype, :ttl, :firstseen, :lastseen, :count); end
 
 	class Client
-		def initialize(pdns=['bfk','certee','dnsparse','dnsdb','virustotal'])
+		def initialize(pdns=['bfk','certee','dnsdb','virustotal','tcpiputils'])
 			@pdnsdbs = []
 			pdns.uniq.each do |pd|
 				case pd
@@ -23,14 +23,16 @@ module PassiveDNS
 					@pdnsdbs << PassiveDNS::BFK.new
 				when 'certee'
 					@pdnsdbs << PassiveDNS::CERTEE.new
-				when 'dnsparse'
-					@pdnsdbs << PassiveDNS::DNSParse.new
+        #when 'dnsparse'
+				#	@pdnsdbs << PassiveDNS::DNSParse.new
 				when 'dnsdb'
 					@pdnsdbs << PassiveDNS::DNSDB.new
 				when 'isc'
 					@pdnsdbs << PassiveDNS::DNSDB.new
 				when 'virustotal'
 					@pdnsdbs << PassiveDNS::VirusTotal.new
+        when 'tcpiputils'
+					@pdnsdbs << PassiveDNS::TCPIPUtils.new
 				else
 					raise "Unknown Passive DNS provider: #{pd}"
 				end
@@ -43,11 +45,11 @@ module PassiveDNS
 			end
 		end
 		
-		def query(item)
+		def query(item, limit=nil)
 			threads = []
 			@pdnsdbs.each do |pdnsdb|
 				threads << Thread.new(item) do |q|
-					pdnsdb.lookup(q)
+					pdnsdb.lookup(q, limit)
 				end
 			end
 				

data/passivedns-client.gemspec

@@ -24,6 +24,6 @@ Gem::Specification.new do |spec|
 	spec.add_development_dependency "bundler", "~> 1.3"
 	spec.add_development_dependency "rake"
 
-	spec.signing_key   = "#{File.dirname(__FILE__)}/../gem-private_key.pem"
-	spec.cert_chain    = ["#{File.dirname(__FILE__)}/../gem-public_cert.pem"]
+	#spec.signing_key   = "#{File.dirname(__FILE__)}/../gem-private_key.pem"
+	#spec.cert_chain    = ["#{File.dirname(__FILE__)}/../gem-public_cert.pem"]
 end

data/test/test_passivedns-client.rb

@@ -15,41 +15,6 @@ class TestPassiveDnsQuery < Test::Unit::TestCase
 		end
 	end
 
-	def test_instantiate_BFK_Client
-		assert_not_nil(PassiveDNS::BFK.new)
-		assert_nothing_raised do
-			PassiveDNS::Client.new(['bfk'])
-		end
-	end
-
-	def test_instantiate_CERTEE_Client
-		assert_not_nil(PassiveDNS::CERTEE.new)
-		assert_nothing_raised do
-			PassiveDNS::Client.new(['certee'])
-		end
-	end
-	
-	def test_instantiate_DNSParse_Client
-		assert_not_nil(PassiveDNS::DNSParse.new)
-		assert_nothing_raised do
-			PassiveDNS::Client.new(['dnsparse'])
-		end
-	end
-	
-	def test_instantiate_DNSDB_Client
-		assert_not_nil(PassiveDNS::DNSDB.new)
-		assert_nothing_raised do
-			PassiveDNS::Client.new(['dnsdb'])
-		end
-	end
-	
-	def test_instantiate_VirusTotal_Client
-		assert_not_nil(PassiveDNS::VirusTotal.new)
-		assert_nothing_raised do
-			PassiveDNS::Client.new(['virustotal'])
-		end
-	end
-
 	def test_instantiate_All_Clients
 		assert_nothing_raised do
 			PassiveDNS::Client.new()
@@ -70,49 +35,80 @@ class TestPassiveDnsQuery < Test::Unit::TestCase
 		end
 	end
 	
-	def test_query_BFK
-		rows = PassiveDNS::BFK.new.lookup("example.org")
+	def test_BFK
+		assert_not_nil(PassiveDNS::BFK.new)
+		assert_nothing_raised do
+			PassiveDNS::Client.new(['bfk'])
+		end
+		rows = PassiveDNS::BFK.new.lookup("example.org",3)
 		assert_not_nil(rows)
 		assert_not_nil(rows.to_s)
 		assert_not_nil(rows.to_xml)
 		assert_not_nil(rows.to_json)
 		assert_not_nil(rows.to_yaml)
+    assert_equal(3, rows.length)
 	end
 	
-	def test_query_CERTEE
-		rows = PassiveDNS::CERTEE.new.lookup("sim.cert.ee")
-		assert_not_nil(rows)
-		assert_not_nil(rows.to_s)
-		assert_not_nil(rows.to_xml)
-		assert_not_nil(rows.to_json)
-		assert_not_nil(rows.to_yaml)
-	end
-
-	def test_query_DNSParse
-		rows = PassiveDNS::DNSParse.new.lookup("example.org")
+	def test_CERTEE
+    assert(false, "CERTEE is still offline")
+		assert_not_nil(PassiveDNS::CERTEE.new)
+		assert_nothing_raised do
+			PassiveDNS::Client.new(['certee'])
+		end
+		rows = PassiveDNS::CERTEE.new.lookup("sim.cert.ee",3)
 		assert_not_nil(rows)
 		assert_not_nil(rows.to_s)
 		assert_not_nil(rows.to_xml)
 		assert_not_nil(rows.to_json)
 		assert_not_nil(rows.to_yaml)
+    assert_equal(3, rows.length)
 	end
 
-	def test_query_DNSDB
-		rows = PassiveDNS::DNSDB.new.lookup("example.org")
+	def test_DNSDB
+		assert_not_nil(PassiveDNS::DNSDB.new)
+		assert_nothing_raised do
+			PassiveDNS::Client.new(['dnsdb'])
+		end
+		rows = PassiveDNS::DNSDB.new.lookup("example.org",3)
 		assert_not_nil(rows)
 		assert_not_nil(rows.to_s)
 		assert_not_nil(rows.to_xml)
 		assert_not_nil(rows.to_json)
 		assert_not_nil(rows.to_yaml)
+    assert_equal(3, rows.length) # this will fail since DNSDB has an off by one error
 	end
 
-	def test_query_VirusTotal
-		rows = PassiveDNS::VirusTotal.new.lookup("sim.cert.ee")
+	def test_VirusTotal
+		assert_not_nil(PassiveDNS::VirusTotal.new)
+		assert_nothing_raised do
+			PassiveDNS::Client.new(['virustotal'])
+		end
+		rows = PassiveDNS::VirusTotal.new.lookup("google.com",3)
 		assert_not_nil(rows)
 		assert_not_nil(rows.to_s)
 		assert_not_nil(rows.to_xml)
 		assert_not_nil(rows.to_json)
 		assert_not_nil(rows.to_yaml)
+    assert_equal(3, rows.length)
 	end
-
+  
+  def test_TCPIPUtils
+    assert_not_nil(PassiveDNS::TCPIPUtils.new)
+    assert_nothing_raised do
+      PassiveDNS::Client.new(['tcpiputils'])
+    end
+    rows = PassiveDNS::TCPIPUtils.new.lookup("example.org")
+    assert_not_nil(rows)
+    assert_not_nil(rows.to_s)
+    assert_not_nil(rows.to_xml)
+    assert_not_nil(rows.to_json)
+    assert_not_nil(rows.to_yaml)
+    rows = PassiveDNS::TCPIPUtils.new.lookup("example.org",3)
+    assert_not_nil(rows)
+    assert_not_nil(rows.to_s)
+    assert_not_nil(rows.to_xml)
+    assert_not_nil(rows.to_json)
+    assert_not_nil(rows.to_yaml)
+    assert_equal(3, rows.length)
+  end
 end

metadata

@@ -1,105 +1,83 @@
 --- !ruby/object:Gem::Specification
 name: passivedns-client
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.3.0
 platform: ruby
 authors:
 - chrislee35
 autorequire: 
 bindir: bin
-cert_chain:
-- |
-  -----BEGIN CERTIFICATE-----
-  MIIDYjCCAkqgAwIBAgIBADANBgkqhkiG9w0BAQUFADBXMREwDwYDVQQDDAhydWJ5
-  Z2VtczEYMBYGCgmSJomT8ixkARkWCGNocmlzbGVlMRMwEQYKCZImiZPyLGQBGRYD
-  ZGhzMRMwEQYKCZImiZPyLGQBGRYDb3JnMB4XDTEzMDUyMjEyNTk0N1oXDTE0MDUy
-  MjEyNTk0N1owVzERMA8GA1UEAwwIcnVieWdlbXMxGDAWBgoJkiaJk/IsZAEZFghj
-  aHJpc2xlZTETMBEGCgmSJomT8ixkARkWA2RoczETMBEGCgmSJomT8ixkARkWA29y
-  ZzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANcPrx8BZiWIR9xWWG8I
-  tqR538tS1t+UJ4FZFl+1vrtU9TiuWX3Vj37TwUpa2fFkziK0n5KupVThyEhcem5m
-  OGRjvgrRFbWQJSSscIKOpwqURHVKRpV9gVz/Hnzk8S+xotUR1Buo3Ugr+I1jHewD
-  Cgr+y+zgZbtjtHsJtsuujkOcPhEjjUinj68L9Fz9BdeJQt+IacjwAzULix6jWCht
-  Uc+g+0z8Esryca2G6I1GsrgX6WHw8dykyQDT9dCtS2flCOwSC1R0K5T/xHW54f+5
-  wcw8mm53KLNe+tmgVC6ZHyME+qJsBnP6uxF0aTEnGA/jDBQDhQNTF0ZP/abzyTsL
-  zjUCAwEAAaM5MDcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAwHQYDVR0OBBYEFO8w
-  +aeP7T6kVJblCg6eusOII9DfMA0GCSqGSIb3DQEBBQUAA4IBAQBCQyRJLXsBo2Fy
-  8W6e/W4RemQRrlAw9DK5O6U71JtedVob2oq+Ob+zmS+PifE2+L+3RiJ2H6VTlOzi
-  x+A061MUXhGraqVq4J2FC8kt4EQywAD0P0Ta5GU24CGSF08Y3GkJy1Sa4XqTC2YC
-  o51s7JP+tkCCtpVYSdzJhTllieRAWBpGV1dtaoeUKE6tYPMBkosxSRcVGczk/Sc3
-  7eQCpexYy9JlUBI9u3BqIY9E+l+MSn8ihXSPmyK0DgrhaCu+voaSFVOX6Y+B5qbo
-  jLXMQu2ZgISYwXNjNbGVHehut82U7U9oiHoWcrOGazaRUmGO9TXP+aJLH0gw2dcK
-  AfMglXPi
-  -----END CERTIFICATE-----
-date: 2013-09-21 00:00:00.000000000 Z
+cert_chain: []
+date: 2014-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.4.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.4.3
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.3.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.3.3
 - !ruby/object:Gem::Dependency
   name: structformatter
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: This provides interfaces to various passive DNS databases to do the query
@@ -112,7 +90,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -122,8 +100,8 @@ files:
 - lib/passivedns/client/bfk.rb
 - lib/passivedns/client/certee.rb
 - lib/passivedns/client/dnsdb.rb
-- lib/passivedns/client/dnsparse.rb
 - lib/passivedns/client/state.rb
+- lib/passivedns/client/tcpiputils.rb
 - lib/passivedns/client/version.rb
 - lib/passivedns/client/virustotal.rb
 - passivedns-client.gemspec
@@ -139,17 +117,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Query passive DNS databases

data/lib/passivedns/client/dnsparse.rb

@@ -1,89 +0,0 @@
-# DESCRIPTION: this is a module for pdns.rb, primarily used by pdnstool.rb, to query Bojan Zdrnja's passive DNS database, DNSParse
-require 'net/http'
-require 'net/https'
-require 'openssl'
-
-module PassiveDNS
-	class DNSParse
-		attr_accessor :debug
-		@@dns_rtypes = {1 => 'A', 2 => 'NS', 3 => 'MD', 4 => 'MF', 5 => 'CNAME', 6 => 'SOA', 7 => 'MB', 8 => 'MG', 9 => 'MR', 10 => 'NULL', 11 => 'WKS', 12 => 'PTR', 13 => 'HINFO', 14 => 'MINFO', 15 => 'MX', 16 => 'TXT', 17 => 'RP', 18 => 'AFSDB', 19 => 'X25', 20 => 'ISDN', 21 => 'RT', 22 => 'NSAP', 23 => 'NSAP-PTR', 24 => 'SIG', 25 => 'KEY', 26 => 'PX', 27 => 'GPOS', 28 => 'AAAA', 29 => 'LOC', 30 => 'NXT', 31 => 'EID', 32 => 'NIMLOC', 33 => 'SRV', 34 => 'ATMA', 35 => 'NAPTR', 36 => 'KX', 37 => 'CERT', 38 => 'A6', 39 => 'DNAME', 40 => 'SINK', 41 => 'OPT', 42 => 'APL', 43 => 'DS', 44 => 'SSHFP', 45 => 'IPSECKEY', 46 => 'RRSIG', 47 => 'NSEC', 48 => 'DNSKEY', 49 => 'DHCID', 55 => 'HIP', 99 => 'SPF', 100 => 'UINFO', 101 => 'UID', 102 => 'GID', 103 => 'UNSPEC', 249 => 'TKEY', 250 => 'TSIG', 251 => 'IXFR', 252 => 'AXFR', 253 => 'MAILB', 254 => 'MAILA', 255 => 'ALL'}
-		def initialize(config="#{ENV['HOME']}/.dnsparse")
-			if File.exist?(config)
-				@base,@user,@pass = File.open(config).read.split(/\n/)
-				$stderr.puts "DEBUG: DNSParse#initialize(#{@base}, #{@user}, #{@pass})" if @debug
-			else
-				raise "Configuration file for DNSParse is required for intialization\nFormat of configuration file (default: #{ENV['HOME']}/.dnsparse) is:\n<url>\n<username>\n<password>\n"
-			end
-		end
-
-		def parse_json(page,response_time=0)
-			res = []
-			# need to remove the json_class tag or the parser will crap itself trying to find a class to align it to
-			page = page.gsub(/\"json_class\"\:\"PDNSResult\"\,/,'')
-			recs = JSON.parse(page)
-			recs.each do |row|
-				res << PDNSResult.new('DNSParse',response_time,row["query"],row["answer"],@@dns_rtypes[row["rrtype"].to_i],row["ttl"],row["firstseen"],row["lastseen"])
-			end
-			res
-		rescue Exception => e
-			$stderr.puts "DNSParse Exception: #{e}"
-			raise e
-		end
-
-		def parse_html(page,response_time=0)
-			rows = []
-			line = page.split(/<table/).grep(/ id=\"dnsparse\"/)
-			return [] unless line.length > 0
-			line = line[0].gsub(/[\t\n]/,'').gsub(/<\/table.*/,'')
-			rows = line.split(/<tr.*?>/)
-			res = []
-			rows.collect do |row|
-				r = row.split(/<td>/).map{|x| x.gsub(/<.*?>/,'').gsub(/\&.*?;/,'').gsub(/[\t\n]/,'')}[1,1000]
-				if r and r[0] =~ /\w/
-					#TXT records screw up other functions and don't provide much without a lot of subparshing. Dropping for now.
-					if r[2]!="TXT" then
-						res << PDNSResult.new('DNSParse',response_time,r[0],r[1],r[2],r[3],r[4],r[5])
-					end
-				end
-			end
-			res
-		rescue Exception => e
-			$stderr.puts "DNSParse Exception: #{e}"
-			raise e
-		end
-
-		def lookup(label)
-			$stderr.puts "DEBUG: DNSParse.lookup(#{label})" if @debug
-			Timeout::timeout(240) {
-				year = Time.now.strftime("%Y").to_i
-				month = Time.now.strftime("%m").to_i
-				url = "#{@base}#{label}&year=#{year - 1}"
-				url = "#{@base}#{label}&year=#{year}" if month > 3
-				if label =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}$/
-					url.gsub!(/query\.php/,'cidr.php')
-				elsif label =~ /\*$/
-					url.gsub!(/query\.php/,'wildcard.php')
-				end
-				$stderr.puts "DEBUG: DNSParse url = #{url}" if @debug
-				url = URI.parse url
-				http = Net::HTTP.new(url.host, url.port)
-				http.use_ssl = (url.scheme == 'https')
-				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-				http.verify_depth = 5
-				request = Net::HTTP::Get.new(url.path+"?"+url.query)
-				request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
-				request.basic_auth @user, @pass
-				t1 = Time.now
-				response = http.request(request)
-				t2 = Time.now
-				if @base =~ /format=json/
-					parse_json(response.body,t2-t1)
-				else
-					parse_html(response.body,t2-t1)
-				end
-			}
-		rescue Timeout::Error => e
-			$stderr.puts "DNSParse lookup timed out: #{label}"
-		end
-	end
-end

data.tar.gz.sig

@@ -1 +0,0 @@
-��mѯm��]�y��!���N�I��̡U�媫=1��:���1:H�L�s��g�����0g�'$�Za�/dX�e�Dϳ�Y*�k���;���q�`��J|�����Usu4L:�?U�'!*j%@?�{Q[��H#c#u��߱�h��ܚ�d�@!�nv�=K�`Ȩ0�'����А���-"Ê�R�t�]M�i�m��2g�
L���澠�rz�ݲ���{���xޢ<I4��P�B�K��"R������

metadata.gz.sig

@@ -1,2 +0,0 @@
-��$Y��]B�Z�y�d������Z��Uy�H(�D������d4肩���j��߄l�s�:�L"�Bq���`D�������۰��
-�𧞇�T_���~k��`�pxc���'��ZjP~c�!E	}վMS�A���H&�m�F���h�=ig�j�X���xd����-0_Ƣ;a_�?Z��pX�t��2�u�j�@l2u�x�d�(�(@R����1�Bܳ�81B��C����M���
8�	�S��