passivedns-client 1.0.0

data/lib/passivedns/client/certee.rb

@@ -0,0 +1,36 @@
+require 'socket'
+
+module PassiveDNS
+	class CERTEE
+		@@host = "sim.cert.ee"
+		attr_accessor :debug
+		def initialize
+		end
+		def lookup(label)
+			$stderr.puts "DEBUG: CERTEE.lookup(#{label})" if @debug
+			recs = []
+			begin
+				t1 = Time.now
+				s = TCPSocket.new(@@host,43)
+				s.puts(label)
+				s.each_line do |l|
+					(lbl,ans,fs,ls) = l.chomp.split(/\t/)
+					rrtype = 'A'
+					if ans =~ /^\d+\.\d+\.\d+\.\d+$/
+						rrtype = 'A'
+					elsif ans =~ /^ns/
+						rrtype = 'NS'
+					else
+						rrtype = 'CNAME'
+					end
+					t2 = Time.now
+					recs << PDNSResult.new('CERTEE',t2-t1,lbl,ans,rrtype,0,Time.parse(fs).utc.strftime("%Y-%m-%dT%H:%M:%SZ"),Time.parse(ls).utc.strftime("%Y-%m-%dT%H:%M:%SZ"))
+				end
+			rescue SocketError => e
+				$stderr.puts e
+			end
+			return nil unless recs.length > 0
+			recs
+		end	
+	end
+end

data/lib/passivedns/client/dnsparse.rb

@@ -0,0 +1,89 @@
+# DESCRIPTION: this is a module for pdns.rb, primarily used by pdnstool.rb, to query Bojan Zdrnja's passive DNS database, DNSParse
+require 'net/http'
+require 'net/https'
+require 'openssl'
+
+module PassiveDNS
+	class DNSParse
+		attr_accessor :debug
+		@@dns_rtypes = {1 => 'A', 2 => 'NS', 3 => 'MD', 4 => 'MF', 5 => 'CNAME', 6 => 'SOA', 7 => 'MB', 8 => 'MG', 9 => 'MR', 10 => 'NULL', 11 => 'WKS', 12 => 'PTR', 13 => 'HINFO', 14 => 'MINFO', 15 => 'MX', 16 => 'TXT', 17 => 'RP', 18 => 'AFSDB', 19 => 'X25', 20 => 'ISDN', 21 => 'RT', 22 => 'NSAP', 23 => 'NSAP-PTR', 24 => 'SIG', 25 => 'KEY', 26 => 'PX', 27 => 'GPOS', 28 => 'AAAA', 29 => 'LOC', 30 => 'NXT', 31 => 'EID', 32 => 'NIMLOC', 33 => 'SRV', 34 => 'ATMA', 35 => 'NAPTR', 36 => 'KX', 37 => 'CERT', 38 => 'A6', 39 => 'DNAME', 40 => 'SINK', 41 => 'OPT', 42 => 'APL', 43 => 'DS', 44 => 'SSHFP', 45 => 'IPSECKEY', 46 => 'RRSIG', 47 => 'NSEC', 48 => 'DNSKEY', 49 => 'DHCID', 55 => 'HIP', 99 => 'SPF', 100 => 'UINFO', 101 => 'UID', 102 => 'GID', 103 => 'UNSPEC', 249 => 'TKEY', 250 => 'TSIG', 251 => 'IXFR', 252 => 'AXFR', 253 => 'MAILB', 254 => 'MAILA', 255 => 'ALL'}
+		def initialize(config="#{ENV['HOME']}/.dnsparse")
+			if File.exist?(config)
+				@base,@user,@pass = File.open(config).read.split(/\n/)
+				$stderr.puts "DEBUG: DNSParse#initialize(#{@base}, #{@user}, #{@pass})" if @debug
+			else
+				raise "Configuration file for DNSParse is required for intialization\nFormat of configuration file (default: #{ENV['HOME']}/.dnsparse) is:\n<url>\n<username>\n<password>\n"
+			end
+		end
+
+		def parse_json(page,response_time=0)
+			res = []
+			# need to remove the json_class tag or the parser will crap itself trying to find a class to align it to
+			page = page.gsub(/\"json_class\"\:\"PDNSResult\"\,/,'')
+			recs = JSON.parse(page)
+			recs.each do |row|
+				res << PDNSResult.new('DNSParse',response_time,row["query"],row["answer"],@@dns_rtypes[row["rrtype"].to_i],row["ttl"],row["firstseen"],row["lastseen"])
+			end
+			res
+		rescue Exception => e
+			$stderr.puts "DNSParse Exception: #{e}"
+			raise e
+		end
+
+		def parse_html(page,response_time=0)
+			rows = []
+			line = page.split(/<table/).grep(/ id=\"dnsparse\"/)
+			return [] unless line.length > 0
+			line = line[0].gsub(/[\t\n]/,'').gsub(/<\/table.*/,'')
+			rows = line.split(/<tr.*?>/)
+			res = []
+			rows.collect do |row|
+				r = row.split(/<td>/).map{|x| x.gsub(/<.*?>/,'').gsub(/\&.*?;/,'').gsub(/[\t\n]/,'')}[1,1000]
+				if r and r[0] =~ /\w/
+					#TXT records screw up other functions and don't provide much without a lot of subparshing. Dropping for now.
+					if r[2]!="TXT" then
+						res << PDNSResult.new('DNSParse',response_time,r[0],r[1],r[2],r[3],r[4],r[5])
+					end
+				end
+			end
+			res
+		rescue Exception => e
+			$stderr.puts "DNSParse Exception: #{e}"
+			raise e
+		end
+
+		def lookup(label)
+			$stderr.puts "DEBUG: DNSParse.lookup(#{label})" if @debug
+			Timeout::timeout(240) {
+				year = Time.now.strftime("%Y").to_i
+				month = Time.now.strftime("%m").to_i
+				url = "#{@base}#{label}&year=#{year - 1}"
+				url = "#{@base}#{label}&year=#{year}" if month > 3
+				if label =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}$/
+					url.gsub!(/query\.php/,'cidr.php')
+				elsif label =~ /\*$/
+					url.gsub!(/query\.php/,'wildcard.php')
+				end
+				$stderr.puts "DEBUG: DNSParse url = #{url}" if @debug
+				url = URI.parse url
+				http = Net::HTTP.new(url.host, url.port)
+				http.use_ssl = (url.scheme == 'https')
+				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+				http.verify_depth = 5
+				request = Net::HTTP::Get.new(url.path+"?"+url.query)
+				request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
+				request.basic_auth @user, @pass
+				t1 = Time.now
+				response = http.request(request)
+				t2 = Time.now
+				if @base =~ /format=json/
+					parse_json(response.body,t2-t1)
+				else
+					parse_html(response.body,t2-t1)
+				end
+			}
+		rescue Timeout::Error => e
+			$stderr.puts "DNSParse lookup timed out: #{label}"
+		end
+	end
+end

data/lib/passivedns/client/isc.rb

@@ -0,0 +1,80 @@
+# DESCRIPTION: this is a module for pdns.rb, primarily used by pdnstool.rb, to query the ISC passive DNS database
+# details on the API are at https://dnsdb.isc.org/doc/isc-dnsdb-api.html
+# to request an API key, please email dnsdb@isc.org
+require 'net/http'
+require 'net/https'
+
+module PassiveDNS
+	class ISC
+		attr_accessor :debug
+		@@base="https://dnsdb-api.isc.org/lookup"
+		
+		def initialize(config="#{ENV['HOME']}/.isc-dnsdb-query.conf")
+			@debug = false
+			if File.exist?(config)
+				@key = File.open(config).readline.chomp
+				if @key =~ /^[0-9a-f]{64}$/
+					# pass
+				elsif @key =~ /^APIKEY=\"([0-9a-f]{64})\"/
+					@key = $1
+				else
+					raise "Format of configuration file (default: #{ENV['HOME']}/.isc-dnsdb-query.conf) is:\nAPIKEY=\"<key>\"\nE.g.,\nAPIKEY=\"d41d8cd98f00b204e9800998ecf8427ed41d8cd98f00b204e9800998ecf8427e\"\n"
+				end
+			else
+				raise "Configuration file for ISC is required for intialization\nFormat of configuration file (default: #{ENV['HOME']}/.isc-dnsdb-query.conf) is:\nAPIKEY=\"<key>\"\nE.g.,\nAPIKEY=\"d41d8cd98f00b204e9800998ecf8427ed41d8cd98f00b204e9800998ecf8427e\"\n"
+			end
+		end
+
+		def parse_json(page,response_time)
+			res = []
+			raise "Error: unable to parse request" if page =~ /Error: unable to parse request/
+			# need to remove the json_class tag or the parser will crap itself trying to find a class to align it to
+			rows = page.split(/\n/)
+			rows.each do |row|
+				record = JSON.parse(row)
+				record['rdata'] = [record['rdata']] if record['rdata'].class == String
+				record['rdata'].each do |rdata|
+					if record['time_first']
+						res << PDNSResult.new('ISC',response_time,record['rrname'],rdata,record['rrtype'],0,Time.at(record['time_first'].to_i).utc.strftime("%Y-%m-%dT%H:%M:%SZ"),Time.at(record['time_last'].to_i).utc.strftime("%Y-%m-%dT%H:%M:%SZ"),record['count'])
+					else
+						res << PDNSResult.new('ISC',response_time,record['rrname'],rdata,record['rrtype'])
+					end
+				end
+			end
+			res
+		rescue Exception => e
+			$stderr.puts "ISC Exception: #{e}"
+			$stderr.puts page
+			raise e
+		end
+
+		def lookup(label)
+			$stderr.puts "DEBUG: ISC.lookup(#{label})" if @debug
+			Timeout::timeout(240) {
+				url = nil
+				if label =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(\/\d{1,2})?$/
+					label = label.gsub(/\//,',')
+					url = "#{@@base}/rdata/ip/#{label}"
+				else
+					url = "#{@@base}/rrset/name/#{label}"
+				end
+				url = URI.parse url
+				http = Net::HTTP.new(url.host, url.port)
+				http.use_ssl = (url.scheme == 'https')
+				http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+				http.verify_depth = 5
+				request = Net::HTTP::Get.new(url.path)
+				request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
+				request.add_field("X-API-Key", @key)
+				request.add_field("Accept", "application/json")
+				t1 = Time.now
+				response = http.request(request)
+				t2 = Time.now
+				$stderr.puts response if @debug
+				parse_json(response.body,t2-t1)
+			}
+		rescue Timeout::Error => e
+			$stderr.puts "ISC lookup timed out: #{label}"
+		end
+	end
+end

data/lib/passivedns/client/state.rb

@@ -0,0 +1,272 @@
+require 'sqlite3'
+require 'yaml'
+require 'structformatter'
+
+module PassiveDNS
+	class PDNSQueueEntry < Struct.new(:query, :state, :level); end
+
+	class PDNSToolState
+		attr_accessor :debug
+		attr_reader :level
+
+		def initialize
+			@queue = []
+			@recs = []
+			@level = 0
+		end
+
+		def next_result
+			@recs.each do |rec|
+				yield rec
+			end
+		end
+
+		def add_result(res)
+			@recs << res
+			add_query(res.answer,'pending')
+			add_query(res.query,'pending')
+		end
+
+		def update_query(query,state)
+			@queue.each do |q|
+				if q.query == query
+					puts "update_query: #{query} (#{q.state}) -> (#{state})" if @debug
+					q.state = state
+					break
+				end
+			end
+		end
+
+		def get_state(query)
+			@queue.each do |q|
+				if q.query == query
+					return q.state
+				end
+			end
+			false
+		end
+
+		def add_query(query,state,level=@level+1)
+			if query =~ /^\d+ \w+\./
+				query = query.split(/ /,2)[1]
+			end
+			return if get_state(query)
+			puts "Adding query: #{query}, #{state}, #{level}" if @debug
+			@queue << PDNSQueueEntry.new(query,state,level)
+		end
+
+		def each_query(max_level=20)
+			@queue.each do |q|
+				if q.state == 'pending' or q.state == 'failed'
+					@level = q.level
+					q.state = 'queried'
+					if q.level < max_level
+						yield q.query
+					end
+				end
+			end
+		end
+
+		def to_gdf
+			output = "nodedef> name,description VARCHAR(12),color,style\n"
+			# IP "$node2,,white,1"
+			# domain "$node2,,gray,2"
+			# Struct.new(:query, :answer, :rrtype, :ttl, :firstseen, :lastseen)
+			colors = {"MX" => "green", "A" => "blue", "CNAME" => "pink", "NS" => "red", "SOA" => "white", "PTR" => "purple", "TXT" => "brown"}
+			nodes = {}
+			edges = {}
+			next_result do |i|
+				if i 
+					nodes[i.query + ",,gray,2"] = true
+					if i.answer =~ /[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/ then
+						nodes[i.answer + ",,white,1"] = true
+					else	
+						nodes[i.answer + ",,gray,2"] = true
+					end
+					color = colors[i.rrtype]
+					color ||= "blue"
+					edges[i.query + "," + i.answer + "," + color] = true
+				end
+			end
+			nodes.each do |i,j|
+				output += i+"\n"
+			end
+			output += "edgedef> node1,node2,color\n"
+			edges.each do |i,j|
+				output += i+"\n"
+			end
+			output
+		end
+
+		def to_graphviz
+			colors = {"MX" => "green", "A" => "blue", "CNAME" => "pink", "NS" => "red", "SOA" => "white", "PTR" => "purple", "TXT" => "brown"}
+			output = "graph pdns {\n"
+			nodes = {}
+			next_result do |l|
+				if l
+					unless nodes[l.query]
+						output += "  \"#{l.query}\" [shape=ellipse, style=filled, color=gray];\n"
+						if l.answer =~ /^\d{3}\.\d{3}\.\d{3}\.\d{3}$/
+							output += "  \"#{l.answer}\" [shape=box, style=filled, color=white];\n"
+						else
+							output += "  \"#{l.answer}\" [shape=ellipse, style=filled, color=gray];\n"
+						end
+						nodes[l.query] = true
+					end
+					output += "  \"#{l.query}\" -- \"#{l.answer}\" [color=#{colors[l.rrtype]}];\n"
+				end
+			end
+			output += "}\n"
+		end
+
+		def to_graphml
+			output = '<?xml version="1.0" encoding="UTF-8"?>
+<graphml xmlns="http://graphml.graphdrawing.org/xmlns"  
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns
+     http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
+  <graph id="G" edgedefault="directed">
+'
+			nodes = {}
+			edges = {}
+			next_result do |r|
+				if r
+					output += "    <node id='#{r.query}'/>\n" unless nodes["#{r.query}"]
+					nodes[r.query] = true
+					output += "    <node id='#{r.answer}'/>\n" unless nodes["#{r.answer}"]
+					nodes[r.answer] = true
+					output += "    <edge source='#{r.query}' target='#{r.answer}'/>\n" unless edges["#{r.query}|#{r.answer}"]
+				end
+			end
+			output += '</graph></graphml>'+"\n"
+		end
+
+		def to_xml
+			output = '<?xml version="1.0" encoding="UTF-8" ?>'+"\n"
+			output +=  "<report>\n"
+			output +=  "	<results>\n"
+			next_result do |rec|
+				output +=  "		"+rec.to_xml+"\n"
+			end
+			output +=  "	</results>\n"
+			output +=  "</report>\n"
+		end
+
+		def to_yaml
+			output = ""
+			next_result do |rec|
+				output += rec.to_yaml+"\n"
+			end
+			output
+		end
+
+		def to_json
+			output = "[\n"
+			sep = ""
+			next_result do |rec|
+				output += sep
+				output += rec.to_json
+				sep = ",\n"
+			end
+			output += "\n]\n"
+		end
+
+		def to_s(sep="\t")
+			output = ""
+			next_result do |rec|
+				output += rec.to_s(sep)+"\n"
+			end
+			output
+		end
+	end # class PDNSToolState
+
+
+	class PDNSToolStateDB < PDNSToolState
+		attr_reader :level
+		def initialize(sqlitedb=nil)
+			puts "PDNSToolState  initialize  #{sqlitedb}" if @debug
+			@level = 0
+			@sqlitedb = sqlitedb
+			raise "Cannot use this class without a database file" unless @sqlitedb
+			unless File.exists?(@sqlitedb)
+				newdb = true
+			end
+			@sqlitedbh = SQLite3::Database.new(@sqlitedb)
+			if newdb
+				create_tables
+			end
+			res = @sqlitedbh.execute("select min(level) from queue where state = 'pending'")
+			if res
+				res.each do |row|
+					@level = row[0].to_i
+					puts "changed @level = #{@level}" if @debug
+				end
+			end
+		end
+
+		def create_tables
+			puts "creating tables" if @debug
+			@sqlitedbh.execute("create table results (query, answer, rrtype, ttl, firstseen, lastseen, ts REAL)")
+			@sqlitedbh.execute("create table queue (query, state, level INTEGER, ts REAL)")
+			@sqlitedbh.execute("create index residx on results (ts)")
+			@sqlitedbh.execute("create unique index queue_unique on queue (query)")
+			@sqlitedbh.execute("create index queue_level_idx on queue (level)")
+			@sqlitedbh.execute("create index queue_state_idx on queue (state)")
+		end
+
+		def next_result
+			rows = @sqlitedbh.execute("select query, answer, rrtype, ttl, firstseen, lastseen from results order by ts")
+			rows.each do |row|
+				yield PDNSResult.new(*row)
+			end
+		end
+
+		def add_result(res)
+			puts "adding result: #{res.to_s}" if @debug
+			curtime = Time.now().to_f
+			@sqlitedbh.execute("insert into results values ('#{res.query}','#{res.answer}','#{res.rrtype}','#{res.ttl}','#{res.firstseen}','#{res.lastseen}',#{curtime})")
+
+			add_query(res.answer,'pending')
+			add_query(res.query,'pending')
+		end
+
+		def add_query(query,state,level=@level+1)
+			return if get_state(query)
+			curtime = Time.now().to_f
+			begin
+				puts "add_query(#{query},#{state},level=#{level})" if @debug
+				@sqlitedbh.execute("insert into queue values ('#{query}','#{state}',#{level},#{curtime})")
+			rescue
+			end
+		end
+
+		def update_query(query,state)
+			@sqlitedbh.execute("update queue set state = '#{state}' where query = '#{query}'")
+		end
+
+		def get_state(query)
+			rows = @sqlitedbh.execute("select state from queue where query = '#{query}'")
+			if rows
+				rows.each do |row|
+					return row[0]
+				end
+			end
+			false
+		end
+
+		def each_query(max_level=20)
+			puts "each_query max_level=#{max_level} curlevel=#{@level}" if @debug
+			rows = @sqlitedbh.execute("select query, state, level from queue where state = 'failed' or state = 'pending' order by level limit 1")
+			if rows
+				rows.each do |row|
+					query,state,level = row
+					puts "  #{query},#{state},#{level}" if @debug
+					if level < max_level
+						update_query(query,'queried')
+						yield query
+					end
+				end
+			end
+		end
+	end # class PDNSToolStateDB
+end