passivedns-client 2.1.6 → 2.1.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/passivedns/client.rb +68 -36
- data/lib/passivedns/client/cli.rb +139 -139
- data/lib/passivedns/client/provider/bfk.rb +54 -52
- data/lib/passivedns/client/provider/circl.rb +39 -39
- data/lib/passivedns/client/provider/cn360.rb +21 -21
- data/lib/passivedns/client/provider/dnsdb.rb +56 -56
- data/lib/passivedns/client/provider/mnemonic.rb +62 -55
- data/lib/passivedns/client/provider/passivetotal.rb +43 -43
- data/lib/passivedns/client/provider/riskiq.rb +40 -40
- data/lib/passivedns/client/provider/tcpiputils.rb +18 -18
- data/lib/passivedns/client/provider/virustotal.rb +46 -46
- data/lib/passivedns/client/state.rb +236 -236
- data/lib/passivedns/client/version.rb +1 -1
- metadata +2 -2
|
@@ -5,7 +5,7 @@ module PassiveDNS #:nodoc: don't document this
|
|
|
5
5
|
module Provider
|
|
6
6
|
|
|
7
7
|
# Queries BFK.de's passive DNS database
|
|
8
|
-
|
|
8
|
+
class BFK < PassiveDB
|
|
9
9
|
# Sets the modules self-reported name to "BFK.de"
|
|
10
10
|
def self.name
|
|
11
11
|
"BFK.de"
|
|
@@ -34,72 +34,74 @@ module PassiveDNS #:nodoc: don't document this
|
|
|
34
34
|
#
|
|
35
35
|
# PassiveDNS::Provider::BFK.new(options)
|
|
36
36
|
#
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
@debug = options[:debug] || false
|
|
37
|
+
def initialize(options={})
|
|
38
|
+
@debug = options[:debug] || false
|
|
40
39
|
@base = options["URL"] || "http://www.bfk.de/bfk_dnslogger.html?query="
|
|
41
|
-
|
|
40
|
+
end
|
|
42
41
|
|
|
43
42
|
# Takes a label (either a domain or an IP address) and returns
|
|
44
43
|
# an array of PassiveDNS::PDNSResult instances with the answers to the query
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
44
|
+
def lookup(label, limit=nil)
|
|
45
|
+
$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
|
|
46
|
+
Timeout::timeout(240) {
|
|
47
|
+
t1 = Time.now
|
|
48
|
+
open(
|
|
49
|
+
@base+label,
|
|
50
|
+
"User-Agent" => "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}"
|
|
51
|
+
) do |f|
|
|
52
|
+
t2 = Time.now
|
|
53
|
+
recs = parse(f.read,t2-t1)
|
|
55
54
|
if limit
|
|
56
55
|
recs[0,limit]
|
|
57
56
|
else
|
|
58
57
|
recs
|
|
59
58
|
end
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
59
|
+
end
|
|
60
|
+
}
|
|
61
|
+
rescue Timeout::Error => e
|
|
62
|
+
$stderr.puts "#{self.class.name} lookup timed out: #{label}"
|
|
63
|
+
end
|
|
65
64
|
|
|
66
65
|
private
|
|
67
66
|
|
|
68
67
|
# parses the webpage returned by BFK to generate an array of PDNSResult
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
68
|
+
def parse(page,response_time)
|
|
69
|
+
line = page.unpack('C*').pack('U*').split(/<table/).grep(/ id=\"logger\"/)
|
|
70
|
+
return [] unless line.length > 0
|
|
71
|
+
line = line[0].gsub(/[\t\n]/,'').gsub(/<\/table.*/,'')
|
|
72
|
+
rows = line.split(/<tr.*?>/)
|
|
73
|
+
res = []
|
|
74
|
+
rows.collect do |row|
|
|
75
|
+
r = row.split(/<td>/).map{|x| x.gsub(/<.*?>/,'').gsub(/\&.*?;/,'')}[1,1000]
|
|
76
|
+
if r and r[0] =~ /\w/
|
|
77
|
+
# BFK includes the MX weight in the answer response. First, find the MX records,
|
|
78
|
+
# then dump the weight to present a consistent record name to the collecting
|
|
79
|
+
# array. Otherwise the other repositories will present the same answer and
|
|
80
|
+
# your results will become cluttered with duplicates.
|
|
81
|
+
if r[1] == "MX" then
|
|
82
|
+
# MX lines look like "5 mx.domain.tld", so split on the space and assign r[2] (:answer) to the latter part.
|
|
83
|
+
#s = r[2].split(/\w/).map{|x| x}[1,1000]
|
|
84
|
+
# r[2] = s[1]
|
|
85
|
+
r[2] =~ /[0-9]+?\s(.+)/
|
|
86
|
+
s=$1
|
|
87
|
+
#puts "DEBUG: == BFK: MX Parsing Strip: Answer: " + r[2] + " : mod: " + s if @debug
|
|
88
|
+
r[2] = s
|
|
89
|
+
|
|
90
|
+
######### FIX BLANKS FOR MX
|
|
91
|
+
|
|
92
|
+
end
|
|
91
93
|
query = r[0]
|
|
92
94
|
answer = r[2]
|
|
93
95
|
rrtype = r[1]
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
96
|
+
res << PDNSResult.new(self.class.name,response_time,query,answer,rrtype,'white')
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
res
|
|
100
|
+
rescue Exception => e
|
|
101
|
+
$stderr.puts "#{self.class.name} Exception: #{e}"
|
|
102
|
+
raise e
|
|
103
|
+
end
|
|
102
104
|
|
|
103
|
-
|
|
105
|
+
end
|
|
104
106
|
end
|
|
105
|
-
end
|
|
107
|
+
end
|
|
@@ -9,7 +9,7 @@ module PassiveDNS #:nodoc: don't document this
|
|
|
9
9
|
module Provider
|
|
10
10
|
# Queries CIRCL.LU's passive DNS database
|
|
11
11
|
# Circl is aliased by CIRCL
|
|
12
|
-
|
|
12
|
+
class Circl < PassiveDB
|
|
13
13
|
# Sets the modules self-reported name to "CIRCL"
|
|
14
14
|
def self.name
|
|
15
15
|
"CIRCL"
|
|
@@ -45,68 +45,68 @@ module PassiveDNS #:nodoc: don't document this
|
|
|
45
45
|
#
|
|
46
46
|
# PassiveDNS::Provider::CIRCL.new(options)
|
|
47
47
|
#
|
|
48
|
-
|
|
48
|
+
def initialize(options={})
|
|
49
49
|
@debug = options[:debug] || false
|
|
50
50
|
@username = options["USERNAME"]
|
|
51
51
|
@password = options["PASSWORD"]
|
|
52
52
|
@auth_token = options["AUTH_TOKEN"]
|
|
53
53
|
@url = options["URL"] || "https://www.circl.lu/pdns/query"
|
|
54
|
-
|
|
54
|
+
end
|
|
55
55
|
|
|
56
56
|
# Takes a label (either a domain or an IP address) and returns
|
|
57
57
|
# an array of PassiveDNS::PDNSResult instances with the answers to the query
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
58
|
+
def lookup(label, limit=nil)
|
|
59
|
+
$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
|
|
60
|
+
Timeout::timeout(240) {
|
|
61
|
+
url = @url+"/"+label
|
|
62
|
+
$stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
|
|
63
|
+
url = URI.parse url
|
|
64
|
+
http = Net::HTTP.new(url.host, url.port)
|
|
65
|
+
http.use_ssl = (url.scheme == 'https')
|
|
66
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
|
67
|
+
http.verify_depth = 5
|
|
68
|
+
request = Net::HTTP::Get.new(url.request_uri)
|
|
69
|
+
request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
|
|
70
70
|
if @username
|
|
71
71
|
request.basic_auth(@username, @password)
|
|
72
72
|
end
|
|
73
73
|
if @auth_token
|
|
74
74
|
request.add_field("Authorization", @auth_token)
|
|
75
75
|
end
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
76
|
+
t1 = Time.now
|
|
77
|
+
response = http.request(request)
|
|
78
|
+
t2 = Time.now
|
|
79
|
+
recs = parse_json(response.body, label, t2-t1)
|
|
80
|
+
if limit
|
|
81
|
+
recs[0,limit]
|
|
82
|
+
else
|
|
83
|
+
recs
|
|
84
|
+
end
|
|
85
|
+
}
|
|
86
|
+
rescue Timeout::Error => e
|
|
87
|
+
$stderr.puts "#{self.class.name} lookup timed out: #{label}"
|
|
88
|
+
end
|
|
89
89
|
|
|
90
90
|
private
|
|
91
91
|
|
|
92
92
|
# parses the response of circl's JSON reply to generate an array of PDNSResult
|
|
93
|
-
|
|
94
|
-
|
|
93
|
+
def parse_json(page,query,response_time=0)
|
|
94
|
+
res = []
|
|
95
95
|
page.split(/\n/).each do |line|
|
|
96
96
|
row = JSON.parse(line)
|
|
97
97
|
firstseen = Time.at(row['time_first'].to_i)
|
|
98
98
|
lastseen = Time.at(row['time_last'].to_i)
|
|
99
|
-
|
|
99
|
+
res << PDNSResult.new(self.class.name,response_time,
|
|
100
100
|
row['rrname'], row['rdata'], row['rrtype'], 0,
|
|
101
|
-
firstseen, lastseen, row['count'])
|
|
101
|
+
firstseen, lastseen, row['count'], 'yellow')
|
|
102
102
|
end
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
103
|
+
res
|
|
104
|
+
rescue Exception => e
|
|
105
|
+
$stderr.puts "#{self.class.name} Exception: #{e}"
|
|
106
|
+
raise e
|
|
107
|
+
end
|
|
108
108
|
|
|
109
|
-
|
|
109
|
+
end
|
|
110
110
|
CIRCL = PassiveDNS::Provider::Circl
|
|
111
111
|
end
|
|
112
|
-
end
|
|
112
|
+
end
|
|
@@ -61,22 +61,22 @@ module PassiveDNS #:nodoc: don't document this
|
|
|
61
61
|
limit ||= 10000
|
|
62
62
|
path = "/api/#{table}/keyword/#{label}/count/#{limit}/"
|
|
63
63
|
url = @cp["API"]+path
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
64
|
+
url = URI.parse url
|
|
65
|
+
http = Net::HTTP.new(url.host, url.port)
|
|
66
|
+
http.use_ssl = (url.scheme == 'https')
|
|
67
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE # I hate doing this
|
|
68
|
+
http.verify_depth = 5
|
|
69
|
+
request = Net::HTTP::Get.new(url.path)
|
|
70
|
+
request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
|
|
71
71
|
request.add_field('Accept', 'application/json')
|
|
72
72
|
request.add_field("X-BashTokid", @cp["API_ID"])
|
|
73
73
|
token = Digest::MD5.hexdigest(path+@cp["API_KEY"])
|
|
74
|
-
|
|
74
|
+
$stderr.puts "DEBUG: cn360 url = #{url} token = #{token}" if @debug
|
|
75
75
|
request.add_field("X-BashToken", token)
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
76
|
+
t1 = Time.now
|
|
77
|
+
response = http.request(request)
|
|
78
|
+
t2 = Time.now
|
|
79
|
+
recs = parse_json(response.body, label, t2-t1)
|
|
80
80
|
if limit
|
|
81
81
|
recs[0,limit]
|
|
82
82
|
else
|
|
@@ -88,8 +88,8 @@ module PassiveDNS #:nodoc: don't document this
|
|
|
88
88
|
|
|
89
89
|
# parses the response of 360.cn's JSON reply to generate an array of PDNSResult
|
|
90
90
|
def parse_json(page,query,response_time=0)
|
|
91
|
-
|
|
92
|
-
|
|
91
|
+
res = []
|
|
92
|
+
data = JSON.parse(page)
|
|
93
93
|
data.each do |row|
|
|
94
94
|
time_first = (row["time_first"]) ? Time.at(row["time_first"].to_i) : nil
|
|
95
95
|
time_last = (row["time_last"]) ? Time.at(row["time_last"].to_i) : nil
|
|
@@ -98,14 +98,14 @@ module PassiveDNS #:nodoc: don't document this
|
|
|
98
98
|
answers = row["rdata"].gsub(/;$/,'').split(/;/)
|
|
99
99
|
rrtype = row["rrtype"]
|
|
100
100
|
answers.each do |answer|
|
|
101
|
-
res << PDNSResult.new(self.class.name, response_time, query, answer, rrtype, time_first, time_last, count)
|
|
101
|
+
res << PDNSResult.new(self.class.name, response_time, query, answer, rrtype, time_first, time_last, count, 'yellow')
|
|
102
102
|
end
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
103
|
+
end
|
|
104
|
+
res
|
|
105
|
+
rescue Exception => e
|
|
106
|
+
$stderr.puts "#{self.class.name} Exception: #{e}"
|
|
107
|
+
raise e
|
|
108
108
|
end
|
|
109
109
|
end
|
|
110
110
|
end
|
|
111
|
-
end
|
|
111
|
+
end
|
|
@@ -8,7 +8,7 @@ module PassiveDNS #:nodoc: don't document this
|
|
|
8
8
|
# The Provider module contains all the Passive DNS provider client code
|
|
9
9
|
module Provider
|
|
10
10
|
# Queries FarSight's passive DNS database
|
|
11
|
-
|
|
11
|
+
class DNSDB < PassiveDB
|
|
12
12
|
# Sets the modules self-reported name to "DNSDB"
|
|
13
13
|
def self.name
|
|
14
14
|
"DNSDB"
|
|
@@ -39,80 +39,80 @@ module PassiveDNS #:nodoc: don't document this
|
|
|
39
39
|
#
|
|
40
40
|
# PassiveDNS::Provider::DNSDB.new(options)
|
|
41
41
|
#
|
|
42
|
-
|
|
43
|
-
|
|
42
|
+
def initialize(options={})
|
|
43
|
+
@debug = options[:debug] || false
|
|
44
44
|
@key = options["APIKEY"] || raise("APIKEY option required for #{self.class}")
|
|
45
45
|
@base = options["URL"] || "https://api.dnsdb.info/lookup"
|
|
46
|
-
|
|
46
|
+
end
|
|
47
47
|
|
|
48
48
|
# Takes a label (either a domain or an IP address) and returns
|
|
49
49
|
# an array of PassiveDNS::PDNSResult instances with the answers to the query
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
50
|
+
def lookup(label, limit=nil)
|
|
51
|
+
$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
|
|
52
|
+
Timeout::timeout(240) {
|
|
53
|
+
url = nil
|
|
54
|
+
if label =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(\/\d{1,2})?$/
|
|
55
|
+
label = label.gsub(/\//,',')
|
|
56
|
+
url = "#{@base}/rdata/ip/#{label}"
|
|
57
|
+
else
|
|
58
|
+
url = "#{@base}/rrset/name/#{label}"
|
|
59
|
+
end
|
|
60
|
+
url = URI.parse url
|
|
61
|
+
http = Net::HTTP.new(url.host, url.port)
|
|
62
|
+
http.use_ssl = (url.scheme == 'https')
|
|
63
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
|
64
|
+
http.verify_depth = 5
|
|
65
65
|
path = url.path
|
|
66
66
|
if limit
|
|
67
67
|
path << "?limit=#{limit}"
|
|
68
68
|
end
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
69
|
+
request = Net::HTTP::Get.new(path)
|
|
70
|
+
request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
|
|
71
|
+
request.add_field("X-API-Key", @key)
|
|
72
|
+
request.add_field("Accept", "application/json")
|
|
73
|
+
t1 = Time.now
|
|
74
|
+
response = http.request(request)
|
|
75
|
+
t2 = Time.now
|
|
76
|
+
$stderr.puts response.body if @debug
|
|
77
|
+
parse_json(response.body,t2-t1)
|
|
78
|
+
}
|
|
79
|
+
rescue Timeout::Error => e
|
|
80
|
+
$stderr.puts "#{self.class.name} lookup timed out: #{label}"
|
|
81
|
+
end
|
|
82
82
|
|
|
83
83
|
private
|
|
84
84
|
|
|
85
85
|
# parses the response of DNSDB's JSON reply to generate an array of PDNSResult
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
86
|
+
def parse_json(page,response_time)
|
|
87
|
+
res = []
|
|
88
|
+
raise "Error: unable to parse request" if page =~ /Error: unable to parse request/
|
|
89
|
+
rows = page.split(/\n/)
|
|
90
|
+
rows.each do |row|
|
|
91
|
+
record = JSON.parse(row)
|
|
92
92
|
answers = record['rdata']
|
|
93
|
-
|
|
93
|
+
answers = [record['rdata']] if record['rdata'].class == String
|
|
94
94
|
query = record['rrname'].gsub!(/\.$/,'')
|
|
95
95
|
rrtype = record['rrtype']
|
|
96
96
|
firstseen = Time.at(record['time_first'].to_i)
|
|
97
97
|
lastseen = Time.at(record['time_last'].to_i)
|
|
98
98
|
count = record['count']
|
|
99
99
|
|
|
100
|
-
|
|
100
|
+
answers.each do |answer|
|
|
101
101
|
answer.gsub!(/\.$/,'')
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
0,firstseen,lastseen,count)
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
102
|
+
if record['time_first']
|
|
103
|
+
res << PDNSResult.new(self.class.name,response_time,query,answer,rrtype,
|
|
104
|
+
0,firstseen,lastseen,count, 'yellow')
|
|
105
|
+
else
|
|
106
|
+
res << PDNSResult.new(self.class.name,response_time,query,answer,rrtype)
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
res
|
|
111
|
+
rescue Exception => e
|
|
112
|
+
$stderr.puts "#{self.class.name} Exception: #{e}"
|
|
113
|
+
$stderr.puts page
|
|
114
|
+
raise e
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
117
|
end
|
|
118
|
-
end
|
|
118
|
+
end
|