passive-dns 0.1.0

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Chris Lee, PhD
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,19 @@
+= passive-dns
+
+Description goes here.
+
+== Contributing to passive-dns
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2011 Chris Lee, PhD. See LICENSE.txt for
+further details.
+

data/bin/pdnstool

@@ -0,0 +1,192 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'passive-dns'
+require 'yaml'
+
+#if __FILE__ == $0
+	def pdnslookup(state,pdnsdbs,recursedepth=1,wait=0,debug=false)
+		puts "pdnslookup: #{state.level} #{recursedepth}" if debug
+		level = 0
+		while level < recursedepth
+			puts "pdnslookup: #{level} < #{recursedepth}" if debug
+			state.each_query(recursedepth) do |q|
+				pdnsdbs.each do |pdnsdb|
+					rv = pdnsdb.lookup(q)
+					if rv
+						rv.each do |r|
+							if ["A","AAAA","NS","CNAME","PTR"].index(r.rrtype)
+								puts "pdnslookup: #{r.to_s}" if debug
+								state.add_result(r)
+							end
+						end
+					else
+						state.update_query(rv,'failed')
+					end
+				end
+				sleep wait if level < recursedepth
+			end
+			level += 1
+		end
+		state
+	end
+	
+	def printresults(state,format,sep="\t")
+		case format
+		when 'text'
+			puts state.to_s(sep)
+		when 'yaml'
+			puts state.to_yaml
+		when 'xml'
+			puts state.to_xml
+		when 'json'
+			puts state.to_json
+		when 'gdf'
+			puts state.to_gdf
+		when 'graphviz'
+			puts state.to_graphviz
+		when 'graphml'
+			puts state.to_graphml
+		end
+	end
+
+	def usage
+		puts "Usage: #{$0} [-a|-b|-d|-i|-e] [-c|-x|-y|-j|-t] [-s <sep>] [-f <file>] [-r#|-w#|-l] <ip|domain|cidr>"
+		puts "  -a uses all of the available passive dns databases"
+		puts "  -b only use BFK"
+		puts "  -d only use DNSParse (default)"
+		puts "  -i only use ISC"
+		puts "  -e only use CERT-EE"
+		puts "  -g outputs a link-nodal GDF visualization definition"
+		puts "  -v outputs a link-nodal graphviz visualization definition"
+		puts "  -m output a link-nodal graphml visualization definition"
+		puts "  -c outputs CSV"
+		puts "  -x outputs XML"
+		puts "  -y outputs YAML"
+		puts "  -j outputs JSON"
+		puts "  -t outputs ASCII text (default)"
+		puts "  -s <sep> specifies a field separator for text output, default is tab"
+		puts "  -f[file] specifies a sqlite3 database used to read the current state - useful for large result sets and generating graphs of previous runs."
+		puts "  -r# specifies the levels of recursion to pull. **WARNING** This is quite taxing on the pDNS servers, so use judiciously (never more than 3 or so) or find yourself blocked!"
+		puts "  -w# specifies the amount of time to wait, in seconds, between queries (Default: 0)"
+		puts "  -l outputs debugging information"
+		exit
+	end
+	
+	opts = GetoptLong.new(
+		[ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+		[ '--debug', '-l', GetoptLong::NO_ARGUMENT ],
+		[ '--all', '-a', GetoptLong::NO_ARGUMENT ],
+		[ '--bfk', '-b', GetoptLong::NO_ARGUMENT ],
+		[ '--dnsparse', '-d', GetoptLong::NO_ARGUMENT ],
+		[ '--isc', '-i', GetoptLong::NO_ARGUMENT ],
+		[ '--certee', '-e', GetoptLong::NO_ARGUMENT ],
+		[ '--gdf', '-g', GetoptLong::NO_ARGUMENT ],
+		[ '--graphviz', '-v', GetoptLong::NO_ARGUMENT ],
+		[ '--graphml', '-m', GetoptLong::NO_ARGUMENT ],
+		[ '--csv', '-c', GetoptLong::NO_ARGUMENT ],
+		[ '--xml', '-x', GetoptLong::NO_ARGUMENT ],
+		[ '--yaml', '-y', GetoptLong::NO_ARGUMENT ],
+		[ '--json', '-j', GetoptLong::NO_ARGUMENT ],
+		[ '--text', '-t', GetoptLong::NO_ARGUMENT ],
+		[ '--sep', '-s', GetoptLong::REQUIRED_ARGUMENT ],
+		[ '--sqlite3', '-f', GetoptLong::REQUIRED_ARGUMENT ],
+		[ '--recurse', '-r', GetoptLong::REQUIRED_ARGUMENT ],
+		[ '--wait', '-w', GetoptLong::REQUIRED_ARGUMENT ]
+	)
+	
+	# sets the default search methods
+	pdnsdbs = []
+	format = "text"
+	sep = "\t"
+	recursedepth = 1
+	wait = 0
+	res = nil
+	debug = false
+	sqlitedb = nil
+	
+	opts.each do |opt, arg|
+		case opt
+		when '--help'
+			usage
+		when '--debug'
+			debug = true
+			$stderr.puts "Using proxy settings: http_proxy=#{ENV['http_proxy']}, https_proxy=#{ENV['https_proxy']}"
+		when '--all'
+			pdnsdbs << PassiveDNS::BFKClient.new
+			pdnsdbs << PassiveDNS::DNSParse.new
+			pdnsdbs << PassiveDNS::ISC.new
+			pdnsdbs << PassiveDNS::CERTEE.new
+		when '--bfk'
+			pdnsdbs << PassiveDNS::BFKClient.new
+		when '--dnsparse'
+			pdnsdbs << PassiveDNS::DNSParse.new
+		when '--isc'
+			pdnsdbs << PassiveDNS::ISC.new
+		when '--certee'
+			pdnsdbs << PassiveDNS::CERTEE.new
+		when '--gdf'
+			format = 'gdf'
+		when '--graphviz'
+			format = 'graphviz'
+		when '--graphml'
+			format = 'graphml'
+		when '--csv'
+			format = 'text'
+			sep = ','
+		when '--yaml'
+			format = 'yaml'
+		when '--xml'
+			format = 'xml'
+		when '--json'
+			format = 'json'
+		when '--text'
+			format = 'text'
+		when '--sep'
+			sep = arg
+		when '--recurse'
+			recursedepth = arg.to_i
+			if recursedepth > 3
+				$stderr.puts "WARNING:  a recursedepth of > 3 can be abusive, please reconsider: sleeping 60 seconds for sense to come to you (hint: hit CTRL-C)"
+				sleep 60
+			end
+		when '--wait'
+			wait = arg.to_i
+		when '--sqlite3'
+			sqlitedb = arg
+		else
+			usage
+		end
+	end
+	
+	if pdnsdbs.length == 0
+		pdnsdbs << PassiveDNS::DNSParse.new
+	end
+	
+	pdnsdbs.each do |pdnsdb|
+		pdnsdb.debug = true if debug
+		if pdnsdb.class.to_s == "PassiveDNS::BFKClient" and recursedepth > 1 and wait < 60
+			wait = 60
+			$stderr.puts "Enforcing a minimal 60 second wait when using BFK for recursive crawling"
+		end
+	end
+	
+	state = nil
+	if sqlitedb
+		state = PassiveDNS::PDNSToolStateDB.new(sqlitedb)
+	else
+		state = PassiveDNS::PDNSToolState.new
+	end
+	state.debug = true if debug
+	
+	if ARGV.length > 0
+		ARGV.each do |arg|
+			state.add_query(arg,'pending',0)
+		end
+	else
+		$stdin.each_line do |l|
+			state.add_query(l.chomp,'pending',0)
+		end
+	end
+	pdnslookup(state,pdnsdbs,recursedepth,wait,debug)
+	printresults(state,format,sep)
+#end

data/lib/passive-dns.rb

@@ -0,0 +1,5 @@
+require 'pdns/pdns'
+require 'pdns/dnsparse'
+require 'pdns/iscpdns'
+require 'pdns/bfkclient'
+require 'pdns/certeepdns'

data/lib/pdns/bfkclient.rb

@@ -0,0 +1,58 @@
+require 'open-uri'
+
+module PassiveDNS
+	class BFKClient
+		attr_accessor :debug
+		def initialize
+			@debug = false
+		end
+		@@base = "http://www.bfk.de/bfk_dnslogger.html?query="
+		def parse(page,response_time)
+			line = page.split(/<table/).grep(/ id=\"logger\"/)
+			return [] unless line.length > 0
+			line = line[0].gsub(/[\t\n]/,'').gsub(/<\/table.*/,'')
+			rows = line.split(/<tr.*?>/)
+			res = []
+			rows.collect do |row|
+				r = row.split(/<td>/).map{|x| x.gsub(/<.*?>/,'').gsub(/\&.*?;/,'')}[1,1000]
+				if r and r[0] =~ /\w/
+					# BFK includes the MX weight in the answer response. First, find the MX records, then dump the weight to present a consistent record name to the collecting array. Otherwise the other repositories will present the same answer and your results will become cluttered with duplicates.
+					if r[1] == "MX" then
+						# MX lines look like "5 mx.domain.tld", so split on the space and assign r[2] (:answer) to the latter part.
+						#s = r[2].split(/\w/).map{|x| x}[1,1000]
+						#	r[2] = s[1]
+						r[2] =~ /[0-9]+?\s(.+)/
+						s=$1
+						puts "DEBUG: == BFK: MX Parsing Strip: Answer: " + r[2] + " : mod: " + s if @debug
+						r[2] = s
+							
+							######### FIX BLANKS FOR MX
+							
+					end
+					res << PDNSResult.new('BFK',response_time,r[0],r[2],r[1],nil,nil,nil)
+				end
+			end
+			res
+		rescue Exception => e
+			$stderr.puts "BFKClient Exception: #{e}"
+			raise e
+		end
+
+		def lookup(label)	
+			$stderr.puts "DEBUG: BFKClient.lookup(#{label})" if @debug
+			Timeout::timeout(240) {
+				t1 = Time.now
+				open(
+					@@base+label,
+					"User-Agent" => "Ruby/#{RUBY_VERSION} ChrisLee passive dns script",
+					:http_basic_authentication => [@user,@pass]
+				) do |f|
+					t2 = Time.now
+					parse(f.read,t2-t1)
+				end
+			}
+		rescue Timeout::Error => e
+			$stderr.puts "BFK lookup timed out: #{label}"      
+		end
+	end
+end

data/lib/pdns/certeepdns.rb

@@ -0,0 +1,36 @@
+require 'socket'
+
+module PassiveDNS
+	class CERTEE
+		@@host = "sim.cert.ee"
+		attr_accessor :debug
+		def initialize
+		end
+		def lookup(label)
+			$stderr.puts "DEBUG: CERTEE.lookup(#{label})" if @debug
+			recs = []
+			begin
+				t1 = Time.now
+				s = TCPSocket.new(@@host,43)
+				s.puts(label)
+				s.each_line do |l|
+					(lbl,ans,fs,ls) = l.chomp.split(/\t/)
+					rrtype = 'A'
+					if ans =~ /^\d+\.\d+\.\d+\.\d+$/
+						rrtype = 'A'
+					elsif ans =~ /^ns/
+						rrtype = 'NS'
+					else
+						rrtype = 'CNAME'
+					end
+					t2 = Time.now
+					recs << PDNSResult.new('CERTEE',t2-t1,lbl,ans,rrtype,0,Time.parse(fs).utc.strftime("%Y-%m-%dT%H:%M:%SZ"),Time.parse(ls).utc.strftime("%Y-%m-%dT%H:%M:%SZ"))
+				end
+			rescue SocketError => e
+				$stderr.puts e
+			end
+			return nil unless recs.length > 0
+			recs
+		end	
+	end
+end

data/lib/pdns/dnsparse.rb

@@ -0,0 +1,88 @@
+# DESCRIPTION: this is a module for pdns.rb, primarily used by pdnstool.rb, to query Bojan Zdrnja's passive DNS database, DNSParse
+require 'net/http'
+require 'net/https'
+require 'openssl'
+
+module PassiveDNS
+	class DNSParse
+		attr_accessor :debug
+		@@dns_rtypes = {1 => 'A', 2 => 'NS', 3 => 'MD', 4 => 'MF', 5 => 'CNAME', 6 => 'SOA', 7 => 'MB', 8 => 'MG', 9 => 'MR', 10 => 'NULL', 11 => 'WKS', 12 => 'PTR', 13 => 'HINFO', 14 => 'MINFO', 15 => 'MX', 16 => 'TXT', 17 => 'RP', 18 => 'AFSDB', 19 => 'X25', 20 => 'ISDN', 21 => 'RT', 22 => 'NSAP', 23 => 'NSAP-PTR', 24 => 'SIG', 25 => 'KEY', 26 => 'PX', 27 => 'GPOS', 28 => 'AAAA', 29 => 'LOC', 30 => 'NXT', 31 => 'EID', 32 => 'NIMLOC', 33 => 'SRV', 34 => 'ATMA', 35 => 'NAPTR', 36 => 'KX', 37 => 'CERT', 38 => 'A6', 39 => 'DNAME', 40 => 'SINK', 41 => 'OPT', 42 => 'APL', 43 => 'DS', 44 => 'SSHFP', 45 => 'IPSECKEY', 46 => 'RRSIG', 47 => 'NSEC', 48 => 'DNSKEY', 49 => 'DHCID', 55 => 'HIP', 99 => 'SPF', 100 => 'UINFO', 101 => 'UID', 102 => 'GID', 103 => 'UNSPEC', 249 => 'TKEY', 250 => 'TSIG', 251 => 'IXFR', 252 => 'AXFR', 253 => 'MAILB', 254 => 'MAILA', 255 => 'ALL'}
+		def initialize(config="#{ENV['HOME']}/.dnsparse")
+			if File.exist?(config)
+				@base,@user,@pass = File.open(config).read.split(/\n/)
+				$stderr.puts "DEBUG: DNSParse#initialize(#{@base}, #{@user}, #{@pass})" if @debug
+			else
+				raise "Configuration file for DNSParse is required for intialization\nFormat of configuration file (default: #{ENV['HOME']}/.dnsparse) is:\n<url>\n<username>\n<password>\n"
+			end
+		end
+
+		def parse_json(page,response_time=0)
+			res = []
+			# need to remove the json_class tag or the parser will crap itself trying to find a class to align it to
+			page = page.gsub(/\"json_class\"\:\"PDNSResult\"\,/,'')
+			recs = JSON.parse(page)
+			recs.each do |row|
+				res << PDNSResult.new('DNSParse',response_time,row["query"],row["answer"],@@dns_rtypes[row["rrtype"].to_i],row["ttl"],row["firstseen"],row["lastseen"])
+			end
+			res
+		rescue Exception => e
+			$stderr.puts "DNSParse Exception: #{e}"
+			raise e
+		end
+
+		def parse_html(page,response_time=0)
+			rows = []
+			line = page.split(/<table/).grep(/ id=\"dnsparse\"/)
+			return [] unless line.length > 0
+			line = line[0].gsub(/[\t\n]/,'').gsub(/<\/table.*/,'')
+			rows = line.split(/<tr.*?>/)
+			res = []
+			rows.collect do |row|
+				r = row.split(/<td>/).map{|x| x.gsub(/<.*?>/,'').gsub(/\&.*?;/,'').gsub(/[\t\n]/,'')}[1,1000]
+				if r and r[0] =~ /\w/
+					#TXT records screw up other functions and don't provide much without a lot of subparshing. Dropping for now.
+					if r[2]!="TXT" then
+						res << PDNSResult.new('DNSParse',response_time,r[0],r[1],r[2],r[3],r[4],r[5])
+					end
+				end
+			end
+			res
+		rescue Exception => e
+			$stderr.puts "DNSParse Exception: #{e}"
+			raise e
+		end
+
+		def lookup(label)
+			$stderr.puts "DEBUG: DNSParse.lookup(#{label})" if @debug
+			Timeout::timeout(240) {
+				year = Time.now.strftime("%Y")
+				url = "#{@base}#{label}&year=#{year.to_i - 1}"
+				if label =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}$/
+					url.gsub!(/query\.php/,'cidr.php')
+				elsif label =~ /\*$/
+					url.gsub!(/query\.php/,'wildcard.php')
+				end
+				$stderr.puts "DEBUG: DNSParse url = #{url}" if @debug
+				url = URI.parse url
+				http = Net::HTTP.new(url.host, url.port)
+				http.use_ssl = (url.scheme == 'https')
+				http.ca_file = "#{ENV['HOME']}/bin/data/cacert.pem"
+				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+				http.verify_depth = 5
+				request = Net::HTTP::Get.new(url.path+"?"+url.query)
+				request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} ChrisLee passive dns script")
+				request.basic_auth @user, @pass
+				t1 = Time.now
+				response = http.request(request)
+				t2 = Time.now
+				if @base =~ /format=json/
+					parse_json(response.body,t2-t1)
+				else
+					parse_html(response.body,t2-t1)
+				end
+			}
+		rescue Timeout::Error => e
+			$stderr.puts "DNSParse lookup timed out: #{label}"
+		end
+	end
+end

data/lib/pdns/iscpdns.rb

@@ -0,0 +1,79 @@
+# DESCRIPTION: this is a module for pdns.rb, primarily used by pdnstool.rb, to query the ISC passive DNS database
+# details on the API are at https://dnsdb.isc.org/doc/isc-dnsdb-api.html
+# to request an API key, please email dnsdb@isc.org
+require 'net/http'
+require 'net/https'
+
+module PassiveDNS
+	class ISC
+		attr_accessor :debug
+		@@base="https://dnsdb-api.isc.org/lookup"
+		
+		def initialize(config="#{ENV['HOME']}/.isc-dnsdb-query.conf")
+			@debug = false
+			if File.exist?(config)
+				@key = File.open(config).readline.chomp
+				if @key =~ /^[0-9a-f]{64}$/
+					# pass
+				elsif @key =~ /^APIKEY=\"([0-9a-f]{64})\"/
+					@key = $1
+				else
+					raise "Format of configuration file (default: #{ENV['HOME']}/.isc-dnsdb-query.conf) is:\nAPIKEY=\"<key>\"\nE.g.,\nAPIKEY=\"d41d8cd98f00b204e9800998ecf8427ed41d8cd98f00b204e9800998ecf8427e\"\n"
+				end
+			else
+				raise "Configuration file for ISC is required for intialization\nFormat of configuration file (default: #{ENV['HOME']}/.isc-dnsdb-query.conf) is:\nAPIKEY=\"<key>\"\nE.g.,\nAPIKEY=\"d41d8cd98f00b204e9800998ecf8427ed41d8cd98f00b204e9800998ecf8427e\"\n"
+			end
+		end
+
+		def parse_json(page,response_time)
+			res = []
+			raise "Error: unable to parse request" if page =~ /Error: unable to parse request/
+			# need to remove the json_class tag or the parser will crap itself trying to find a class to align it to
+			rows = page.split(/\n/)
+			rows.each do |row|
+				record = JSON.parse(row)
+				record['rdata'] = [record['rdata']] if record['rdata'].class == String
+				record['rdata'].each do |rdata|
+					if record['time_first']
+						res << PDNSResult.new('ISC',response_time,record['rrname'],rdata,record['rrtype'],0,Time.at(record['time_first'].to_i).utc.strftime("%Y-%m-%dT%H:%M:%SZ"),Time.at(record['time_last'].to_i).utc.strftime("%Y-%m-%dT%H:%M:%SZ"))
+					else
+						res << PDNSResult.new('ISC',response_time,record['rrname'],rdata,record['rrtype'],nil,nil,nil)
+					end
+				end
+			end
+			res
+		rescue Exception => e
+			$stderr.puts "ISC Exception: #{e}"
+			$stderr.puts page
+			raise e
+		end
+
+		def lookup(label)
+			$stderr.puts "DEBUG: ISC.lookup(#{label})" if @debug
+			Timeout::timeout(240) {
+				url = nil
+				if label =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(\/\d{1,2})?$/
+					label = label.gsub(/\//,',')
+					url = "#{@@base}/rdata/ip/#{label}"
+				else
+					url = "#{@@base}/rrset/name/#{label}"
+				end
+				url = URI.parse url
+				http = Net::HTTP.new(url.host, url.port)
+				http.use_ssl = (url.scheme == 'https')
+				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+				http.verify_depth = 5
+				request = Net::HTTP::Get.new(url.path)
+				request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} ChrisLee passive dns script")
+				request.add_field("X-API-Key", @key)
+				request.add_field("Accept", "application/json")
+				t1 = Time.now
+				response = http.request(request)
+				t2 = Time.now
+				parse_json(response.body,t2-t1)
+			}
+		rescue Timeout::Error => e
+			$stderr.puts "ISC lookup timed out: #{label}"
+		end
+	end
+end

data/lib/pdns/pdns.rb

@@ -0,0 +1,280 @@
+#!/usr/bin/env ruby
+# DESCRIPTION: queries passive DNS databases 
+# This code is released under the LGPL: http://www.gnu.org/licenses/lgpl-3.0.txt
+# Please note that use of any passive dns database is subject to the terms of use of that passive dns database.  Use of this script in violation of their terms is not encouraged in any way.  Also, please do not add any obfuscation to try to work around their terms of service.  If you need special services, ask the providers for help/permission.
+# Remember, these passive DNS operators are my friends.  I don't want to have a row with them because some asshat used this library to abuse them.
+
+require 'timeout'
+require 'yaml'
+require 'util/structformatter'
+require 'getoptlong'
+require 'sqlite3'
+
+module PassiveDNS
+	class PDNSResult < Struct.new(:source, :response_time, :query, :answer, :rrtype, :ttl, :firstseen, :lastseen); end
+	class PDNSQueueEntry < Struct.new(:query, :state, :level); end
+	class PDNSToolState
+		attr_accessor :debug
+		attr_reader :level
+		
+		def initialize
+			@queue = []
+			@recs = []
+			@level = 0
+		end
+		
+		def next_result
+			@recs.each do |rec|
+				yield rec
+			end
+		end
+		
+		def add_result(res)
+			@recs << res
+			add_query(res.answer,'pending')
+			add_query(res.query,'pending')
+		end
+		
+		def update_query(query,state)
+			@queue.each do |q|
+				if q.query == query
+					puts "update_query: #{query} (#{q.state}) -> (#{state})" if @debug
+					q.state = state
+					break
+				end
+			end
+		end
+		
+		def get_state(query)
+			@queue.each do |q|
+				if q.query == query
+					return q.state
+				end
+			end
+			false
+		end
+		
+		def add_query(query,state,level=@level+1)
+			if query =~ /^\d+ \w+\./
+				query = query.split(/ /,2)[1]
+			end
+			return if get_state(query)
+			puts "Adding query: #{query}, #{state}, #{level}" if @debug
+			@queue << PDNSQueueEntry.new(query,state,level)
+		end
+		
+		def each_query(max_level=20)
+			@queue.each do |q|
+				if q.state == 'pending' or q.state == 'failed'
+					@level = q.level
+					q.state = 'queried'
+					if q.level < max_level
+						yield q.query
+					end
+				end
+			end
+		end
+		
+		def to_gdf
+			output = "nodedef> name,description VARCHAR(12),color,style\n"
+			# IP "$node2,,white,1"
+			# domain "$node2,,gray,2"
+			# Struct.new(:query, :answer, :rrtype, :ttl, :firstseen, :lastseen)
+			colors = {"MX" => "green", "A" => "blue", "CNAME" => "pink", "NS" => "red", "SOA" => "white", "PTR" => "purple", "TXT" => "brown"}
+			nodes = {}
+			edges = {}
+			next_result do |i|
+				if i 
+					nodes[i.query + ",,gray,2"] = true
+					if i.answer =~ /[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/ then
+						nodes[i.answer + ",,white,1"] = true
+					else	
+						nodes[i.answer + ",,gray,2"] = true
+					end
+					color = colors[i.rrtype]
+					color ||= "blue"
+					edges[i.query + "," + i.answer + "," + color] = true
+				end
+			end
+			nodes.each do |i,j|
+				output += i+"\n"
+			end
+			output += "edgedef> node1,node2,color\n"
+			edges.each do |i,j|
+				output += i+"\n"
+			end
+			output
+		end
+
+		def to_graphviz
+			colors = {"MX" => "green", "A" => "blue", "CNAME" => "pink", "NS" => "red", "SOA" => "white", "PTR" => "purple", "TXT" => "brown"}
+			output = "graph pdns {\n"
+			nodes = {}
+			next_result do |l|
+				if l
+					unless nodes[l.query]
+						output += "  \"#{l.query}\" [shape=ellipse, style=filled, color=gray];\n"
+						if l.answer =~ /^\d{3}\.\d{3}\.\d{3}\.\d{3}$/
+							output += "  \"#{l.answer}\" [shape=box, style=filled, color=white];\n"
+						else
+							output += "  \"#{l.answer}\" [shape=ellipse, style=filled, color=gray];\n"
+						end
+						nodes[l.query] = true
+					end
+					output += "  \"#{l.query}\" -- \"#{l.answer}\" [color=#{colors[l.rrtype]}];\n"
+				end
+			end
+			output += "}\n"
+		end
+
+		def to_graphml
+			output = '<?xml version="1.0" encoding="UTF-8"?>
+<graphml xmlns="http://graphml.graphdrawing.org/xmlns"  
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns
+     http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
+  <graph id="G" edgedefault="directed">
+'
+			nodes = {}
+			edges = {}
+			next_result do |r|
+				if r
+					output += "    <node id='#{r.query}'/>\n" unless nodes["#{r.query}"]
+					nodes[r.query] = true
+					output += "    <node id='#{r.answer}'/>\n" unless nodes["#{r.answer}"]
+					nodes[r.answer] = true
+					output += "    <edge source='#{r.query}' target='#{r.answer}'/>\n" unless edges["#{r.query}|#{r.answer}"]
+				end
+			end
+			output += '</graph></graphml>'+"\n"
+		end
+
+		def to_xml
+			output = '<?xml version="1.0" encoding="UTF-8" ?>'+"\n"
+			output +=  "<report>\n"
+			output +=  "	<results>\n"
+			next_result do |rec|
+				output +=  "		"+rec.to_xml+"\n"
+			end
+			output +=  "	</results>\n"
+			output +=  "</report>\n"
+		end
+
+		def to_yaml
+			output = ""
+			next_result do |rec|
+				output += rec.to_yaml+"\n"
+			end
+			output
+		end
+
+		def to_json
+			output = "[\n"
+			sep = ""
+			next_result do |rec|
+				output += sep
+				output += rec.to_json
+				sep = ",\n"
+			end
+			output += "\n]\n"
+		end
+
+		def to_s(sep="\t")
+			output = ""
+			next_result do |rec|
+				output += rec.to_s(sep)+"\n"
+			end
+			output
+		end
+	end # class PDNSToolState
+
+
+	class PDNSToolStateDB < PDNSToolState
+		attr_reader :level
+		def initialize(sqlitedb=nil)
+			puts "PDNSToolState  initialize  #{sqlitedb}" if @debug
+			@level = 0
+			@sqlitedb = sqlitedb
+			raise "Cannot use this class without a database file" unless @sqlitedb
+			unless File.exists?(@sqlitedb)
+				newdb = true
+			end
+			@sqlitedbh = SQLite3::Database.new(@sqlitedb)
+			if newdb
+				create_tables
+			end
+			res = @sqlitedbh.execute("select min(level) from queue where state = 'pending'")
+			if res
+				res.each do |row|
+					@level = row[0].to_i
+					puts "changed @level = #{@level}" if @debug
+				end
+			end
+		end
+
+		def create_tables
+			puts "creating tables" if @debug
+			@sqlitedbh.execute("create table results (query, answer, rrtype, ttl, firstseen, lastseen, ts REAL)")
+			@sqlitedbh.execute("create table queue (query, state, level INTEGER, ts REAL)")
+			@sqlitedbh.execute("create index residx on results (ts)")
+			@sqlitedbh.execute("create unique index queue_unique on queue (query)")
+			@sqlitedbh.execute("create index queue_level_idx on queue (level)")
+			@sqlitedbh.execute("create index queue_state_idx on queue (state)")
+		end
+
+		def next_result
+			rows = @sqlitedbh.execute("select query, answer, rrtype, ttl, firstseen, lastseen from results order by ts")
+			rows.each do |row|
+				yield PDNSResult.new(*row)
+			end
+		end
+
+		def add_result(res)
+			puts "adding result: #{res.to_s}" if @debug
+			curtime = Time.now().to_f
+			@sqlitedbh.execute("insert into results values ('#{res.query}','#{res.answer}','#{res.rrtype}','#{res.ttl}','#{res.firstseen}','#{res.lastseen}',#{curtime})")
+
+			add_query(res.answer,'pending')
+			add_query(res.query,'pending')
+		end
+
+		def add_query(query,state,level=@level+1)
+			return if get_state(query)
+			curtime = Time.now().to_f
+			begin
+				puts "add_query(#{query},#{state},level=#{level})" if @debug
+				@sqlitedbh.execute("insert into queue values ('#{query}','#{state}',#{level},#{curtime})")
+			rescue
+			end
+		end
+
+		def update_query(query,state)
+			@sqlitedbh.execute("update queue set state = '#{state}' where query = '#{query}'")
+		end
+
+		def get_state(query)
+			rows = @sqlitedbh.execute("select state from queue where query = '#{query}'")
+			if rows
+				rows.each do |row|
+					return row[0]
+				end
+			end
+			false
+		end
+
+		def each_query(max_level=20)
+			puts "each_query max_level=#{max_level} curlevel=#{@level}" if @debug
+			rows = @sqlitedbh.execute("select query, state, level from queue where state = 'failed' or state = 'pending' order by level limit 1")
+			if rows
+				rows.each do |row|
+					query,state,level = row
+					puts "  #{query},#{state},#{level}" if @debug
+					if level < max_level
+						update_query(query,'queried')
+						yield query
+					end
+				end
+			end
+		end
+	end # class PDNSToolStateDB
+end

data/lib/util/structformatter.rb

@@ -0,0 +1,163 @@
+#!/usr/bin/env ruby
+# DESCRIPTION: extends ruby Structs to be outputted as yaml, xml, and json. This is meant to be used as a mixin
+require 'json'
+
+class Array
+	def render_xml(element_name, element)
+		str = ""
+		if element.class == Date
+			str = "<#{element_name}>#{element.strftime("%Y-%m-%d")}</#{element_name}>"
+		elsif element.class == Time or element.class == DateTime
+			str = "<#{element_name}>#{element.strftime("%Y-%m-%dT%H:%M:%SZ")}</#{element_name}>"
+		elsif element.kind_of? Struct or element.kind_of? Hash or element.kind_of? Array
+			str = element.to_xml
+		else
+			str = "<#{element_name}>#{element}</#{element_name}>"
+		end
+	end
+	def to_xml
+		str = "<array>"
+		self.each do |item|
+			str += render_xml("element",item)
+		end
+		str += "</array>"
+	end
+	def to(format)
+		case format
+		when 'xml'
+			self.to_xml
+		when 'json'
+			self.to_json
+		when 'string'
+			self.to_s
+		else
+			raise "invalid format: #{format}, use one of xml, json, or string"
+		end
+	end
+end
+
+class Hash
+	def render_xml(element_name, element)
+		str = ""
+		if element.class == Date
+			str = "<#{element_name}>#{element.strftime("%Y-%m-%d")}</#{element_name}>"
+		elsif element.class == Time or element.class == DateTime
+			str = "<#{element_name}>#{element.strftime("%Y-%m-%dT%H:%M:%SZ")}</#{element_name}>"
+		elsif element.kind_of? Struct or element.kind_of? Hash or element.kind_of? Array
+			str = element.to_xml
+		else
+			str = "<#{element_name}>#{element}</#{element_name}>"
+		end
+	end
+	def to_xml
+		str = "<hash>"
+		self.each do |key,value|
+			str += "<element><key>#{key}</key>"
+			str += render_xml("value",value)
+			str += "</element>"
+		end
+		str += "</hash>"
+	end
+	def to(format)
+		case format
+		when 'xml'
+			self.to_xml
+		when 'json'
+			self.to_json
+		when 'string'
+			self.to_s
+		else
+			raise "invalid format: #{format}, use one of xml, json, or string"
+		end
+	end
+end
+
+class Struct
+	@@printclass = true
+	def Struct::printclass=(pc)
+		@@printclass = pc
+	end
+	def render_xml(element_name, element)
+		str = ""
+		if element.class == Date
+			str = "<#{element_name}>#{element.strftime("%Y-%m-%d")}</#{element_name}>"
+		elsif element.class == Time or element.class == DateTime
+			str = "<#{element_name}>#{element.strftime("%Y-%m-%dT%H:%M:%SZ")}</#{element_name}>"
+		elsif element.kind_of? Struct
+			str = element.to_xml
+		elsif element.kind_of? Hash or element.kind_of? Array
+			str = element.to_xml(element_name)
+		else
+			str = "<#{element_name}>#{element}</#{element_name}>"
+		end
+	end
+	def to_xml
+		children = []
+		str = "<#{self.class}"
+		self.members.each do |member|
+			if self[member].class == Array or self[member].class == Hash or self[member].kind_of? Struct
+				children << member
+			elsif self[member].class == Date
+				str += " #{member}='#{self[member].strftime("%Y-%m-%d")}'"
+			elsif self[member].class == Time
+				str += " #{member}='#{self[member].strftime("%Y-%m-%dT%H:%M:%SZ")}'"
+			else
+				str += " #{member}='#{self[member]}'"
+			end
+		end
+		if children.length == 0
+			str += ' />'
+		else
+			str += '>'
+			children.each do |member|
+				if self[member].class == Array
+					str += "<#{member}s>"
+					self[member].each do |item|
+						str += render_xml(member,item)
+					end
+					str += "</#{member}s>"
+				elsif self[member].class == Hash
+					str += "<#{member}s>"
+					self[member].each do |key,value|
+						str += "<HashElement><key>#{key}</key><value>"
+						str += render_xml(member,value)
+						str += "</value></HashElement>"
+					end
+					str += "</#{member}s>"
+				elsif self[member].kind_of? Struct
+					str += self[member].to_xml
+				end
+			end
+			str += "</#{self.class}>"
+		end
+	end
+	
+	def to_json(*a)
+		hash = (@@printclass) ? { 'class' => self.class } : {}
+		self.members.sort.each do |member|
+			hash[member] = self[member]
+		end
+		hash.to_json(*a)
+	end
+	
+	def to(format)
+		case format
+		when 'xml'
+			self.to_xml
+		when 'json'
+			self.to_json
+		when 'string'
+			self.to_s
+		else
+			raise "invalid format: #{format}, use one of xml, json, or string"
+		end
+	end
+	
+	def to_s(sep = " ")
+		self.members.map{ |x| self[x].to_s }.join(sep)
+	end
+	
+	def to_s_header(sep = " ")
+		self.members.map{ |x| x.to_s }.join(sep)
+	end
+end

data/test/helper.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'passive-dns'
+
+class Test::Unit::TestCase
+end

data/test/test_passive-dns.rb

@@ -0,0 +1,70 @@
+require 'helper'
+
+class TestPassiveDnsQuery < Test::Unit::TestCase
+	should "instantiate BFK Client" do
+		assert_not_nil(PassiveDNS::BFKClient.new) 
+	end
+
+	should "instantiate DNSParse Client" do
+		assert_not_nil(PassiveDNS::DNSParse.new)
+	end
+	
+	should "instantiate ISC Client" do
+		assert_not_nil(PassiveDNS::ISC.new)
+	end
+	
+	should "instantiate CERT-EE Client" do
+		assert_not_nil(PassiveDNS::CERTEE.new)
+	end
+	
+	should "instantiate Passive DNS State" do
+		assert_not_nil(PassiveDNS::PDNSToolState.new)
+	end
+	
+	should "instantiate Passive DNS State database" do
+		if File.exists?("test/test.sqlite3")
+			File.unlink("test/test.sqlite3")
+		end
+		assert_not_nil(PassiveDNS::PDNSToolStateDB.new("test/test.sqlite3"))
+		if File.exists?("test/test.sqlite3")
+			File.unlink("test/test.sqlite3")
+		end
+	end
+	
+	should "query BFK" do
+		rows = PassiveDNS::BFKClient.new.lookup("example.org");
+		assert_not_nil(rows)
+		assert_not_nil(rows.to_s)
+		assert_not_nil(rows.to_xml)
+		assert_not_nil(rows.to_json)
+		assert_not_nil(rows.to_yaml)
+	end
+	
+	should "query DNSParse" do
+		rows = PassiveDNS::DNSParse.new.lookup("example.org");
+		assert_not_nil(rows)
+		assert_not_nil(rows.to_s)
+		assert_not_nil(rows.to_xml)
+		assert_not_nil(rows.to_json)
+		assert_not_nil(rows.to_yaml)
+	end
+
+	should "query ISC" do
+		rows = PassiveDNS::ISC.new.lookup("example.org");
+		assert_not_nil(rows)
+		assert_not_nil(rows.to_s)
+		assert_not_nil(rows.to_xml)
+		assert_not_nil(rows.to_json)
+		assert_not_nil(rows.to_yaml)
+	end
+
+	should "query CERTEE" do
+		rows = PassiveDNS::CERTEE.new.lookup("example.org");
+		assert_not_nil(rows)
+		assert_not_nil(rows.to_s)
+		assert_not_nil(rows.to_xml)
+		assert_not_nil(rows.to_json)
+		assert_not_nil(rows.to_yaml)
+	end
+
+end

metadata

@@ -0,0 +1,225 @@
+--- !ruby/object:Gem::Specification 
+name: passive-dns
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Chris Lee
+autorequire: 
+bindir: bin
+cert_chain: 
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIDYjCCAkqgAwIBAgIBADANBgkqhkiG9w0BAQUFADBXMREwDwYDVQQDDAhydWJ5
+  Z2VtczEYMBYGCgmSJomT8ixkARkWCGNocmlzbGVlMRMwEQYKCZImiZPyLGQBGRYD
+  ZGhzMRMwEQYKCZImiZPyLGQBGRYDb3JnMB4XDTExMDIyNzE1MzAxOVoXDTEyMDIy
+  NzE1MzAxOVowVzERMA8GA1UEAwwIcnVieWdlbXMxGDAWBgoJkiaJk/IsZAEZFghj
+  aHJpc2xlZTETMBEGCgmSJomT8ixkARkWA2RoczETMBEGCgmSJomT8ixkARkWA29y
+  ZzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALNM1Hjs6q58sf7Jp64A
+  vEY2cnRWDdFpD8UWpwaJK5kgSHOVgs+0mtszn+YlYjmx8kpmuYpyU4g9mNMImMQe
+  ow8pVsL4QBBK/1Ozgdxrsptk3IiTozMYA+g2I/+WvZSEDu9uHkKe8pvMBEMrg7RJ
+  IN7+jWaPnSzg3DbFwxwOdi+QRw33DjK7oFWcOaaBqWTUpI4epdi/c/FE1I6UWULJ
+  ZF/Uso0Sc2Pp/YuVhuMHGrUbn7zrWWo76nnK4DTLfXFDbZF5lIXT1w6BtIiN6Ho9
+  Rdr/W6663hYUo3WMsUSa3I5+PJXEBKmGHIZ2TNFnoFIRHha2fmm1HC9+BTaKwcO9
+  PLcCAwEAAaM5MDcwCQYDVR0TBAIwADAdBgNVHQ4EFgQURzsNkZo2rv86Ftc+hVww
+  RNICMrwwCwYDVR0PBAQDAgSwMA0GCSqGSIb3DQEBBQUAA4IBAQBRRw/iNA/PdnvW
+  OBoNCSr/IiHOGZqMHgPJwyWs68FhThnLc2EyIkuLTQf98ms1/D3p0XX9JsxazvKT
+  W/in8Mm/R2fkVziSdzqChtw/4Z4bW3c+RF7TgX6SP5cKxNAfKmAPuItcs2Y+7bdS
+  hr/FktVtT2iAmISRnlEbdaTpfl6N2ZWNT83khV6iOs5xRkX/+0e+GgAv9mE6nqr1
+  AkuDXMhposxcnFZUrZ3UtMPEe/JnyP7Vv6pvr3qtZm8FidFZU91+rX/fwdyBU8RP
+  /5l8uLWXXNt1wEbtu4N1I66LwTK2iRrQZE8XtlgZGbxYDFUkiurq3OafF2YwRs6W
+  6yhklP75
+  -----END CERTIFICATE-----
+
+date: 2011-03-06 00:00:00 -05:00
+default_executable: pdnstool
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 1
+        segments: 
+        - 1
+        - 4
+        - 3
+        version: 1.4.3
+  version_requirements: *id001
+  name: json
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency 
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 29
+        segments: 
+        - 1
+        - 3
+        - 3
+        version: 1.3.3
+  version_requirements: *id002
+  name: sqlite3
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency 
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  version_requirements: *id003
+  name: shoulda
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency 
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  version_requirements: *id004
+  name: bundler
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency 
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 1
+        - 5
+        - 2
+        version: 1.5.2
+  version_requirements: *id005
+  name: jeweler
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency 
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  version_requirements: *id006
+  name: rcov
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency 
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">"
+      - !ruby/object:Gem::Version 
+        hash: 1
+        segments: 
+        - 1
+        - 4
+        - 3
+        version: 1.4.3
+  version_requirements: *id007
+  name: json
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency 
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 29
+        segments: 
+        - 1
+        - 3
+        - 3
+        version: 1.3.3
+  version_requirements: *id008
+  name: sqlite3
+  prerelease: false
+  type: :runtime
+description: This provides interfaces to various passive DNS databases to do the query and to normalize the responses.  The query tool also allows for recursive queries, using an SQLite3 database to keep state.
+email: rubygems@chrislee.dhs.org
+executables: 
+- pdnstool
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE.txt
+- README.rdoc
+files: 
+- bin/pdnstool
+- lib/passive-dns.rb
+- lib/pdns/bfkclient.rb
+- lib/pdns/certeepdns.rb
+- lib/pdns/dnsparse.rb
+- lib/pdns/iscpdns.rb
+- lib/pdns/pdns.rb
+- lib/util/structformatter.rb
+- LICENSE.txt
+- README.rdoc
+- test/helper.rb
+- test/test_passive-dns.rb
+has_rdoc: true
+homepage: http://github.com/chrislee35/passive-dns
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.5.0
+signing_key: 
+specification_version: 3
+summary: Query passive DNS databases
+test_files: 
+- test/helper.rb
+- test/test_passive-dns.rb

metadata.gz.sig

@@ -0,0 +1,2 @@
+��Y��pH�o׏�
+v�kb��
І��UH��;��~�KPK�
�����աu�{T�U��z����$��t�`�V�NjO��e