passivedns-client 1.0.0

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in passivedns-client.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 chrislee35
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,64 @@
+# PassiveDNS::Client
+
+This rubygem queries 5 major Passive DNS databases: BFK, CERTEE, DNSParse, ISC, and VirusTotal.
+Passive DNS is a technique where IP to hostname mappings are made by recording the answers of other people's queries.  
+
+There is a tool included, pdnstool, that wraps a lot of the functionality that you would need.
+
+Please note that use of any passive DNS database is subject to the terms of use of that passive DNS database.  Use of this script in violation of their terms is strongly discouraged.  Also, please do not add any obfuscation to try to work around their terms of service.  If you need special services, ask the providers for help/permission.  Remember, these passive DNS operators are my friends.  I don't want to have a row with them because some jerk used this library to abuse them.
+
+If you like this library, please buy the Passive DNS operators a round of beers.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'passivedns-client'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install passivedns-client
+
+## Usage
+
+	require 'passivedns-client'
+	
+	c = PassiveDNS::Client.new(['bfk','isc']) # providers: bfk, dnsparse, certee, isc, virustotal
+	results = c.query("example.com")
+	
+Or use the included tool!
+
+	Usage: bin/pdnstool [-a|-b|-e|-d|-i|-V] [-c|-x|-y|-j|-t] [-s <sep>] [-f <file>] [-r#|-w#|-l] <ip|domain|cidr>
+	  -a uses all of the available passive dns databases
+	  -b only use BFK
+	  -e only use CERT-EE
+	  -d only use DNSParse (default)
+	  -i only use ISC
+	  -V only use VirusTotal
+
+	  -g outputs a link-nodal GDF visualization definition
+	  -v outputs a link-nodal graphviz visualization definition
+	  -m output a link-nodal graphml visualization definition
+	  -c outputs CSV
+	  -x outputs XML
+	  -y outputs YAML
+	  -j outputs JSON
+	  -t outputs ASCII text (default)
+	  -s <sep> specifies a field separator for text output, default is tab
+
+	  -f[file] specifies a sqlite3 database used to read the current state - useful for large result sets and generating graphs of previous runs.
+	  -r# specifies the levels of recursion to pull. **WARNING** This is quite taxing on the pDNS servers, so use judiciously (never more than 3 or so) or find yourself blocked!
+	  -w# specifies the amount of time to wait, in seconds, between queries (Default: 0)
+	  -l outputs debugging information
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,12 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+	t.libs << 'lib'
+	t.test_files = FileList['test/test_*.rb']
+	t.verbose = true
+end
+
+task :default => :test

data/bin/pdnstool

@@ -0,0 +1,202 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'passivedns-client'
+require 'structformatter'
+require 'getoptlong'
+require 'yaml'
+
+#if __FILE__ == $0
+	def pdnslookup(state,pdnsclient,recursedepth=1,wait=0,debug=false)
+		puts "pdnslookup: #{state.level} #{recursedepth}" if debug
+		level = 0
+		while level < recursedepth
+			puts "pdnslookup: #{level} < #{recursedepth}" if debug
+			state.each_query(recursedepth) do |q|
+				rv = pdnsclient.query(q)
+				if rv
+					rv.each do |r|
+						if ["A","AAAA","NS","CNAME","PTR"].index(r.rrtype)
+							puts "pdnslookup: #{r.to_s}" if debug
+							state.add_result(r)
+						end
+					end
+				else
+					state.update_query(rv,'failed')
+				end
+				sleep wait if level < recursedepth
+			end
+			level += 1
+		end
+		state
+	end
+	
+	def printresults(state,format,sep="\t")
+		case format
+		when 'text'
+			puts PassiveDNS::PDNSResult.members.join(sep)
+			puts state.to_s(sep)
+		when 'yaml'
+			puts state.to_yaml
+		when 'xml'
+			puts state.to_xml
+		when 'json'
+			puts state.to_json
+		when 'gdf'
+			puts state.to_gdf
+		when 'graphviz'
+			puts state.to_graphviz
+		when 'graphml'
+			puts state.to_graphml
+		end
+	end
+
+	def usage
+		puts "Usage: #{$0} [-a|-b|-e|-d|-i|-V] [-c|-x|-y|-j|-t] [-s <sep>] [-f <file>] [-r#|-w#|-l] <ip|domain|cidr>"
+		puts "  -a uses all of the available passive dns databases"
+		puts "  -b only use BFK"
+		puts "  -e only use CERT-EE"
+		puts "  -d only use DNSParse (default)"
+		puts "  -i only use ISC"
+		puts "  -V only use VirusTotal"
+		puts ""
+		puts "  -g outputs a link-nodal GDF visualization definition"
+		puts "  -v outputs a link-nodal graphviz visualization definition"
+		puts "  -m output a link-nodal graphml visualization definition"
+		puts "  -c outputs CSV"
+		puts "  -x outputs XML"
+		puts "  -y outputs YAML"
+		puts "  -j outputs JSON"
+		puts "  -t outputs ASCII text (default)"
+		puts "  -s <sep> specifies a field separator for text output, default is tab"
+		puts ""
+		puts "  -f[file] specifies a sqlite3 database used to read the current state - useful for large result sets and generating graphs of previous runs."
+		puts "  -r# specifies the levels of recursion to pull. **WARNING** This is quite taxing on the pDNS servers, so use judiciously (never more than 3 or so) or find yourself blocked!"
+		puts "  -w# specifies the amount of time to wait, in seconds, between queries (Default: 0)"
+		puts "  -l outputs debugging information"
+		exit
+	end
+	
+	opts = GetoptLong.new(
+		[ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+		[ '--debug', '-l', GetoptLong::NO_ARGUMENT ],
+		[ '--all', '-a', GetoptLong::NO_ARGUMENT ],
+		[ '--bfk', '-b', GetoptLong::NO_ARGUMENT ],
+		[ '--dnsparse', '-d', GetoptLong::NO_ARGUMENT ],
+		[ '--certee', '-e', GetoptLong::NO_ARGUMENT ],
+		[ '--isc', '-i', GetoptLong::NO_ARGUMENT ],
+		[ '--virustotal', '-V', GetoptLong::NO_ARGUMENT ],
+		
+		[ '--gdf', '-g', GetoptLong::NO_ARGUMENT ],
+		[ '--graphviz', '-v', GetoptLong::NO_ARGUMENT ],
+		[ '--graphml', '-m', GetoptLong::NO_ARGUMENT ],
+		[ '--csv', '-c', GetoptLong::NO_ARGUMENT ],
+		[ '--xml', '-x', GetoptLong::NO_ARGUMENT ],
+		[ '--yaml', '-y', GetoptLong::NO_ARGUMENT ],
+		[ '--json', '-j', GetoptLong::NO_ARGUMENT ],
+		[ '--text', '-t', GetoptLong::NO_ARGUMENT ],
+		[ '--sep', '-s', GetoptLong::REQUIRED_ARGUMENT ],
+		
+		[ '--sqlite3', '-f', GetoptLong::REQUIRED_ARGUMENT ],
+		[ '--recurse', '-r', GetoptLong::REQUIRED_ARGUMENT ],
+		[ '--wait', '-w', GetoptLong::REQUIRED_ARGUMENT ]
+	)
+	
+	# sets the default search methods
+	pdnsdbs = []
+	format = "text"
+	sep = "\t"
+	recursedepth = 1
+	wait = 0
+	res = nil
+	debug = false
+	sqlitedb = nil
+	
+	opts.each do |opt, arg|
+		case opt
+		when '--help'
+			usage
+		when '--debug'
+			debug = true
+			$stderr.puts "Using proxy settings: http_proxy=#{ENV['http_proxy']}, https_proxy=#{ENV['https_proxy']}"
+		when '--all'
+			pdnsdbs << "bfk"
+			pdnsdbs << "certee"
+			pdnsdbs << "dnsparse"
+			pdnsdbs << "isc"
+			pdnsdbs << "virustotal"
+		when '--bfk'
+			pdnsdbs << "bfk"
+		when '--certee'
+			pdnsdbs << "certee"
+		when '--dnsparse'
+			pdnsdbs << "dnsparse"
+		when '--isc'
+			pdnsdbs << "isc"
+		when '--virustotal'
+			pdnsdbs << "virustotal"
+		when '--gdf'
+			format = 'gdf'
+		when '--graphviz'
+			format = 'graphviz'
+		when '--graphml'
+			format = 'graphml'
+		when '--csv'
+			format = 'text'
+			sep = ','
+		when '--yaml'
+			format = 'yaml'
+		when '--xml'
+			format = 'xml'
+		when '--json'
+			format = 'json'
+		when '--text'
+			format = 'text'
+		when '--sep'
+			sep = arg
+		when '--recurse'
+			recursedepth = arg.to_i
+			if recursedepth > 3
+				$stderr.puts "WARNING:  a recursedepth of > 3 can be abusive, please reconsider: sleeping 60 seconds for sense to come to you (hint: hit CTRL-C)"
+				sleep 60
+			end
+		when '--wait'
+			wait = arg.to_i
+		when '--sqlite3'
+			sqlitedb = arg
+		else
+			usage
+		end
+	end
+	
+	if pdnsdbs.length == 0
+		pdnsdbs << "dnsparse"
+	end
+	
+	if pdnsdbs.index("bfk") and recursedepth > 1 and wait < 60
+		wait = 60
+		$stderr.puts "Enforcing a minimal 60 second wait when using BFK for recursive crawling"
+	end
+		
+	state = nil
+	if sqlitedb
+		state = PassiveDNS::PDNSToolStateDB.new(sqlitedb)
+	else
+		state = PassiveDNS::PDNSToolState.new
+	end
+	state.debug = true if debug
+	
+	pdnsclient = PassiveDNS::Client.new(pdnsdbs)
+	pdnsclient.debug = debug
+	
+	if ARGV.length > 0
+		ARGV.each do |arg|
+			state.add_query(arg,'pending',0)
+		end
+	else
+		$stdin.each_line do |l|
+			state.add_query(l.chomp,'pending',0)
+		end
+	end
+	pdnslookup(state,pdnsclient,recursedepth,wait,debug)
+	printresults(state,format,sep)
+#end

data/lib/passivedns/client.rb

@@ -0,0 +1,68 @@
+require "passivedns/client/version"
+# DESCRIPTION: queries passive DNS databases 
+# This code is released under the LGPL: http://www.gnu.org/licenses/lgpl-3.0.txt
+# Please note that use of any passive dns database is subject to the terms of use of that passive dns database.  Use of this script in violation of their terms is not encouraged in any way.  Also, please do not add any obfuscation to try to work around their terms of service.  If you need special services, ask the providers for help/permission.
+# Remember, these passive DNS operators are my friends.  I don't want to have a row with them because some asshat used this library to abuse them.
+require 'passivedns/client/bfk.rb'
+require 'passivedns/client/certee.rb'
+require 'passivedns/client/dnsparse.rb'
+require 'passivedns/client/isc.rb'
+require 'passivedns/client/virustotal.rb'
+require 'passivedns/client/state.rb'
+
+module PassiveDNS
+
+	class PDNSResult < Struct.new(:source, :response_time, :query, :answer, :rrtype, :ttl, :firstseen, :lastseen, :count); end
+
+	class Client
+		def initialize(pdns=['bfk','certee','dnsparse','isc','virustotal'])
+			@pdnsdbs = []
+			pdns.uniq.each do |pd|
+				case pd
+				when 'bfk'
+					@pdnsdbs << PassiveDNS::BFK.new
+				when 'certee'
+					@pdnsdbs << PassiveDNS::CERTEE.new
+				when 'dnsparse'
+					@pdnsdbs << PassiveDNS::DNSParse.new
+				when 'isc'
+					@pdnsdbs << PassiveDNS::ISC.new
+				when 'virustotal'
+					@pdnsdbs << PassiveDNS::VirusTotal.new
+				else
+					raise "Unknown Passive DNS provider: #{pd}"
+				end
+			end
+		end #initialize
+		
+		def debug=(d)
+			@pdnsdbs.each do |pdnsdb|
+				pdnsdb.debug = d
+			end
+		end
+		
+		def query(item)
+			threads = []
+			@pdnsdbs.each do |pdnsdb|
+				threads << Thread.new(item) do |q|
+					pdnsdb.lookup(q)
+				end
+			end
+				
+			results = []
+			threads.each do |thr|
+				rv = thr.join.value
+				if rv
+					rv.each do |r|
+						if ["A","AAAA","NS","CNAME","PTR"].index(r.rrtype)
+							results << r
+						end
+					end
+				end
+			end
+			
+			return results
+		end #query
+		
+	end # Client
+end # PassiveDNS

data/lib/passivedns/client/bfk.rb

@@ -0,0 +1,58 @@
+require 'open-uri'
+
+module PassiveDNS
+	class BFK
+		attr_accessor :debug
+		def initialize
+			@debug = false
+		end
+		@@base = "http://www.bfk.de/bfk_dnslogger.html?query="
+		def parse(page,response_time)
+			line = page.unpack('C*').pack('U*').split(/<table/).grep(/ id=\"logger\"/)
+			return [] unless line.length > 0
+			line = line[0].gsub(/[\t\n]/,'').gsub(/<\/table.*/,'')
+			rows = line.split(/<tr.*?>/)
+			res = []
+			rows.collect do |row|
+				r = row.split(/<td>/).map{|x| x.gsub(/<.*?>/,'').gsub(/\&.*?;/,'')}[1,1000]
+				if r and r[0] =~ /\w/
+					# BFK includes the MX weight in the answer response. First, find the MX records, then dump the weight to present a consistent record name to the collecting array. Otherwise the other repositories will present the same answer and your results will become cluttered with duplicates.
+					if r[1] == "MX" then
+						# MX lines look like "5 mx.domain.tld", so split on the space and assign r[2] (:answer) to the latter part.
+						#s = r[2].split(/\w/).map{|x| x}[1,1000]
+						#	r[2] = s[1]
+						r[2] =~ /[0-9]+?\s(.+)/
+						s=$1
+						puts "DEBUG: == BFK: MX Parsing Strip: Answer: " + r[2] + " : mod: " + s if @debug
+						r[2] = s
+							
+							######### FIX BLANKS FOR MX
+							
+					end
+					res << PDNSResult.new('BFK',response_time,r[0],r[2],r[1])
+				end
+			end
+			res
+		rescue Exception => e
+			$stderr.puts "BFKClient Exception: #{e}"
+			raise e
+		end
+
+		def lookup(label)	
+			$stderr.puts "DEBUG: BFKClient.lookup(#{label})" if @debug
+			Timeout::timeout(240) {
+				t1 = Time.now
+				open(
+					@@base+label,
+					"User-Agent" => "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}",
+					:http_basic_authentication => [@user,@pass]
+				) do |f|
+					t2 = Time.now
+					parse(f.read,t2-t1)
+				end
+			}
+		rescue Timeout::Error => e
+			$stderr.puts "BFK lookup timed out: #{label}"      
+		end
+	end
+end