passivedns-client 2.1.6 → 2.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/passivedns/client.rb +68 -36
- data/lib/passivedns/client/cli.rb +139 -139
- data/lib/passivedns/client/provider/bfk.rb +54 -52
- data/lib/passivedns/client/provider/circl.rb +39 -39
- data/lib/passivedns/client/provider/cn360.rb +21 -21
- data/lib/passivedns/client/provider/dnsdb.rb +56 -56
- data/lib/passivedns/client/provider/mnemonic.rb +62 -55
- data/lib/passivedns/client/provider/passivetotal.rb +43 -43
- data/lib/passivedns/client/provider/riskiq.rb +40 -40
- data/lib/passivedns/client/provider/tcpiputils.rb +18 -18
- data/lib/passivedns/client/provider/virustotal.rb +46 -46
- data/lib/passivedns/client/state.rb +236 -236
- data/lib/passivedns/client/version.rb +1 -1
- metadata +2 -2
|
@@ -5,7 +5,7 @@ module PassiveDNS #:nodoc: don't document this
|
|
|
5
5
|
module Provider
|
|
6
6
|
|
|
7
7
|
# Queries BFK.de's passive DNS database
|
|
8
|
-
|
|
8
|
+
class BFK < PassiveDB
|
|
9
9
|
# Sets the modules self-reported name to "BFK.de"
|
|
10
10
|
def self.name
|
|
11
11
|
"BFK.de"
|
|
@@ -34,72 +34,74 @@ module PassiveDNS #:nodoc: don't document this
|
|
|
34
34
|
#
|
|
35
35
|
# PassiveDNS::Provider::BFK.new(options)
|
|
36
36
|
#
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
@debug = options[:debug] || false
|
|
37
|
+
def initialize(options={})
|
|
38
|
+
@debug = options[:debug] || false
|
|
40
39
|
@base = options["URL"] || "http://www.bfk.de/bfk_dnslogger.html?query="
|
|
41
|
-
|
|
40
|
+
end
|
|
42
41
|
|
|
43
42
|
# Takes a label (either a domain or an IP address) and returns
|
|
44
43
|
# an array of PassiveDNS::PDNSResult instances with the answers to the query
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
44
|
+
def lookup(label, limit=nil)
|
|
45
|
+
$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
|
|
46
|
+
Timeout::timeout(240) {
|
|
47
|
+
t1 = Time.now
|
|
48
|
+
open(
|
|
49
|
+
@base+label,
|
|
50
|
+
"User-Agent" => "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}"
|
|
51
|
+
) do |f|
|
|
52
|
+
t2 = Time.now
|
|
53
|
+
recs = parse(f.read,t2-t1)
|
|
55
54
|
if limit
|
|
56
55
|
recs[0,limit]
|
|
57
56
|
else
|
|
58
57
|
recs
|
|
59
58
|
end
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
59
|
+
end
|
|
60
|
+
}
|
|
61
|
+
rescue Timeout::Error => e
|
|
62
|
+
$stderr.puts "#{self.class.name} lookup timed out: #{label}"
|
|
63
|
+
end
|
|
65
64
|
|
|
66
65
|
private
|
|
67
66
|
|
|
68
67
|
# parses the webpage returned by BFK to generate an array of PDNSResult
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
68
|
+
def parse(page,response_time)
|
|
69
|
+
line = page.unpack('C*').pack('U*').split(/<table/).grep(/ id=\"logger\"/)
|
|
70
|
+
return [] unless line.length > 0
|
|
71
|
+
line = line[0].gsub(/[\t\n]/,'').gsub(/<\/table.*/,'')
|
|
72
|
+
rows = line.split(/<tr.*?>/)
|
|
73
|
+
res = []
|
|
74
|
+
rows.collect do |row|
|
|
75
|
+
r = row.split(/<td>/).map{|x| x.gsub(/<.*?>/,'').gsub(/\&.*?;/,'')}[1,1000]
|
|
76
|
+
if r and r[0] =~ /\w/
|
|
77
|
+
# BFK includes the MX weight in the answer response. First, find the MX records,
|
|
78
|
+
# then dump the weight to present a consistent record name to the collecting
|
|
79
|
+
# array. Otherwise the other repositories will present the same answer and
|
|
80
|
+
# your results will become cluttered with duplicates.
|
|
81
|
+
if r[1] == "MX" then
|
|
82
|
+
# MX lines look like "5 mx.domain.tld", so split on the space and assign r[2] (:answer) to the latter part.
|
|
83
|
+
#s = r[2].split(/\w/).map{|x| x}[1,1000]
|
|
84
|
+
# r[2] = s[1]
|
|
85
|
+
r[2] =~ /[0-9]+?\s(.+)/
|
|
86
|
+
s=$1
|
|
87
|
+
#puts "DEBUG: == BFK: MX Parsing Strip: Answer: " + r[2] + " : mod: " + s if @debug
|
|
88
|
+
r[2] = s
|
|
89
|
+
|
|
90
|
+
######### FIX BLANKS FOR MX
|
|
91
|
+
|
|
92
|
+
end
|
|
91
93
|
query = r[0]
|
|
92
94
|
answer = r[2]
|
|
93
95
|
rrtype = r[1]
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
96
|
+
res << PDNSResult.new(self.class.name,response_time,query,answer,rrtype,'white')
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
res
|
|
100
|
+
rescue Exception => e
|
|
101
|
+
$stderr.puts "#{self.class.name} Exception: #{e}"
|
|
102
|
+
raise e
|
|
103
|
+
end
|
|
102
104
|
|
|
103
|
-
|
|
105
|
+
end
|
|
104
106
|
end
|
|
105
|
-
end
|
|
107
|
+
end
|
|
@@ -9,7 +9,7 @@ module PassiveDNS #:nodoc: don't document this
|
|
|
9
9
|
module Provider
|
|
10
10
|
# Queries CIRCL.LU's passive DNS database
|
|
11
11
|
# Circl is aliased by CIRCL
|
|
12
|
-
|
|
12
|
+
class Circl < PassiveDB
|
|
13
13
|
# Sets the modules self-reported name to "CIRCL"
|
|
14
14
|
def self.name
|
|
15
15
|
"CIRCL"
|
|
@@ -45,68 +45,68 @@ module PassiveDNS #:nodoc: don't document this
|
|
|
45
45
|
#
|
|
46
46
|
# PassiveDNS::Provider::CIRCL.new(options)
|
|
47
47
|
#
|
|
48
|
-
|
|
48
|
+
def initialize(options={})
|
|
49
49
|
@debug = options[:debug] || false
|
|
50
50
|
@username = options["USERNAME"]
|
|
51
51
|
@password = options["PASSWORD"]
|
|
52
52
|
@auth_token = options["AUTH_TOKEN"]
|
|
53
53
|
@url = options["URL"] || "https://www.circl.lu/pdns/query"
|
|
54
|
-
|
|
54
|
+
end
|
|
55
55
|
|
|
56
56
|
# Takes a label (either a domain or an IP address) and returns
|
|
57
57
|
# an array of PassiveDNS::PDNSResult instances with the answers to the query
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
58
|
+
def lookup(label, limit=nil)
|
|
59
|
+
$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
|
|
60
|
+
Timeout::timeout(240) {
|
|
61
|
+
url = @url+"/"+label
|
|
62
|
+
$stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
|
|
63
|
+
url = URI.parse url
|
|
64
|
+
http = Net::HTTP.new(url.host, url.port)
|
|
65
|
+
http.use_ssl = (url.scheme == 'https')
|
|
66
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
|
67
|
+
http.verify_depth = 5
|
|
68
|
+
request = Net::HTTP::Get.new(url.request_uri)
|
|
69
|
+
request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
|
|
70
70
|
if @username
|
|
71
71
|
request.basic_auth(@username, @password)
|
|
72
72
|
end
|
|
73
73
|
if @auth_token
|
|
74
74
|
request.add_field("Authorization", @auth_token)
|
|
75
75
|
end
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
76
|
+
t1 = Time.now
|
|
77
|
+
response = http.request(request)
|
|
78
|
+
t2 = Time.now
|
|
79
|
+
recs = parse_json(response.body, label, t2-t1)
|
|
80
|
+
if limit
|
|
81
|
+
recs[0,limit]
|
|
82
|
+
else
|
|
83
|
+
recs
|
|
84
|
+
end
|
|
85
|
+
}
|
|
86
|
+
rescue Timeout::Error => e
|
|
87
|
+
$stderr.puts "#{self.class.name} lookup timed out: #{label}"
|
|
88
|
+
end
|
|
89
89
|
|
|
90
90
|
private
|
|
91
91
|
|
|
92
92
|
# parses the response of circl's JSON reply to generate an array of PDNSResult
|
|
93
|
-
|
|
94
|
-
|
|
93
|
+
def parse_json(page,query,response_time=0)
|
|
94
|
+
res = []
|
|
95
95
|
page.split(/\n/).each do |line|
|
|
96
96
|
row = JSON.parse(line)
|
|
97
97
|
firstseen = Time.at(row['time_first'].to_i)
|
|
98
98
|
lastseen = Time.at(row['time_last'].to_i)
|
|
99
|
-
|
|
99
|
+
res << PDNSResult.new(self.class.name,response_time,
|
|
100
100
|
row['rrname'], row['rdata'], row['rrtype'], 0,
|
|
101
|
-
firstseen, lastseen, row['count'])
|
|
101
|
+
firstseen, lastseen, row['count'], 'yellow')
|
|
102
102
|
end
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
103
|
+
res
|
|
104
|
+
rescue Exception => e
|
|
105
|
+
$stderr.puts "#{self.class.name} Exception: #{e}"
|
|
106
|
+
raise e
|
|
107
|
+
end
|
|
108
108
|
|
|
109
|
-
|
|
109
|
+
end
|
|
110
110
|
CIRCL = PassiveDNS::Provider::Circl
|
|
111
111
|
end
|
|
112
|
-
end
|
|
112
|
+
end
|
|
@@ -61,22 +61,22 @@ module PassiveDNS #:nodoc: don't document this
|
|
|
61
61
|
limit ||= 10000
|
|
62
62
|
path = "/api/#{table}/keyword/#{label}/count/#{limit}/"
|
|
63
63
|
url = @cp["API"]+path
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
64
|
+
url = URI.parse url
|
|
65
|
+
http = Net::HTTP.new(url.host, url.port)
|
|
66
|
+
http.use_ssl = (url.scheme == 'https')
|
|
67
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE # I hate doing this
|
|
68
|
+
http.verify_depth = 5
|
|
69
|
+
request = Net::HTTP::Get.new(url.path)
|
|
70
|
+
request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
|
|
71
71
|
request.add_field('Accept', 'application/json')
|
|
72
72
|
request.add_field("X-BashTokid", @cp["API_ID"])
|
|
73
73
|
token = Digest::MD5.hexdigest(path+@cp["API_KEY"])
|
|
74
|
-
|
|
74
|
+
$stderr.puts "DEBUG: cn360 url = #{url} token = #{token}" if @debug
|
|
75
75
|
request.add_field("X-BashToken", token)
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
76
|
+
t1 = Time.now
|
|
77
|
+
response = http.request(request)
|
|
78
|
+
t2 = Time.now
|
|
79
|
+
recs = parse_json(response.body, label, t2-t1)
|
|
80
80
|
if limit
|
|
81
81
|
recs[0,limit]
|
|
82
82
|
else
|
|
@@ -88,8 +88,8 @@ module PassiveDNS #:nodoc: don't document this
|
|
|
88
88
|
|
|
89
89
|
# parses the response of 360.cn's JSON reply to generate an array of PDNSResult
|
|
90
90
|
def parse_json(page,query,response_time=0)
|
|
91
|
-
|
|
92
|
-
|
|
91
|
+
res = []
|
|
92
|
+
data = JSON.parse(page)
|
|
93
93
|
data.each do |row|
|
|
94
94
|
time_first = (row["time_first"]) ? Time.at(row["time_first"].to_i) : nil
|
|
95
95
|
time_last = (row["time_last"]) ? Time.at(row["time_last"].to_i) : nil
|
|
@@ -98,14 +98,14 @@ module PassiveDNS #:nodoc: don't document this
|
|
|
98
98
|
answers = row["rdata"].gsub(/;$/,'').split(/;/)
|
|
99
99
|
rrtype = row["rrtype"]
|
|
100
100
|
answers.each do |answer|
|
|
101
|
-
res << PDNSResult.new(self.class.name, response_time, query, answer, rrtype, time_first, time_last, count)
|
|
101
|
+
res << PDNSResult.new(self.class.name, response_time, query, answer, rrtype, time_first, time_last, count, 'yellow')
|
|
102
102
|
end
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
103
|
+
end
|
|
104
|
+
res
|
|
105
|
+
rescue Exception => e
|
|
106
|
+
$stderr.puts "#{self.class.name} Exception: #{e}"
|
|
107
|
+
raise e
|
|
108
108
|
end
|
|
109
109
|
end
|
|
110
110
|
end
|
|
111
|
-
end
|
|
111
|
+
end
|
|
@@ -8,7 +8,7 @@ module PassiveDNS #:nodoc: don't document this
|
|
|
8
8
|
# The Provider module contains all the Passive DNS provider client code
|
|
9
9
|
module Provider
|
|
10
10
|
# Queries FarSight's passive DNS database
|
|
11
|
-
|
|
11
|
+
class DNSDB < PassiveDB
|
|
12
12
|
# Sets the modules self-reported name to "DNSDB"
|
|
13
13
|
def self.name
|
|
14
14
|
"DNSDB"
|
|
@@ -39,80 +39,80 @@ module PassiveDNS #:nodoc: don't document this
|
|
|
39
39
|
#
|
|
40
40
|
# PassiveDNS::Provider::DNSDB.new(options)
|
|
41
41
|
#
|
|
42
|
-
|
|
43
|
-
|
|
42
|
+
def initialize(options={})
|
|
43
|
+
@debug = options[:debug] || false
|
|
44
44
|
@key = options["APIKEY"] || raise("APIKEY option required for #{self.class}")
|
|
45
45
|
@base = options["URL"] || "https://api.dnsdb.info/lookup"
|
|
46
|
-
|
|
46
|
+
end
|
|
47
47
|
|
|
48
48
|
# Takes a label (either a domain or an IP address) and returns
|
|
49
49
|
# an array of PassiveDNS::PDNSResult instances with the answers to the query
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
50
|
+
def lookup(label, limit=nil)
|
|
51
|
+
$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
|
|
52
|
+
Timeout::timeout(240) {
|
|
53
|
+
url = nil
|
|
54
|
+
if label =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(\/\d{1,2})?$/
|
|
55
|
+
label = label.gsub(/\//,',')
|
|
56
|
+
url = "#{@base}/rdata/ip/#{label}"
|
|
57
|
+
else
|
|
58
|
+
url = "#{@base}/rrset/name/#{label}"
|
|
59
|
+
end
|
|
60
|
+
url = URI.parse url
|
|
61
|
+
http = Net::HTTP.new(url.host, url.port)
|
|
62
|
+
http.use_ssl = (url.scheme == 'https')
|
|
63
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
|
64
|
+
http.verify_depth = 5
|
|
65
65
|
path = url.path
|
|
66
66
|
if limit
|
|
67
67
|
path << "?limit=#{limit}"
|
|
68
68
|
end
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
69
|
+
request = Net::HTTP::Get.new(path)
|
|
70
|
+
request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
|
|
71
|
+
request.add_field("X-API-Key", @key)
|
|
72
|
+
request.add_field("Accept", "application/json")
|
|
73
|
+
t1 = Time.now
|
|
74
|
+
response = http.request(request)
|
|
75
|
+
t2 = Time.now
|
|
76
|
+
$stderr.puts response.body if @debug
|
|
77
|
+
parse_json(response.body,t2-t1)
|
|
78
|
+
}
|
|
79
|
+
rescue Timeout::Error => e
|
|
80
|
+
$stderr.puts "#{self.class.name} lookup timed out: #{label}"
|
|
81
|
+
end
|
|
82
82
|
|
|
83
83
|
private
|
|
84
84
|
|
|
85
85
|
# parses the response of DNSDB's JSON reply to generate an array of PDNSResult
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
86
|
+
def parse_json(page,response_time)
|
|
87
|
+
res = []
|
|
88
|
+
raise "Error: unable to parse request" if page =~ /Error: unable to parse request/
|
|
89
|
+
rows = page.split(/\n/)
|
|
90
|
+
rows.each do |row|
|
|
91
|
+
record = JSON.parse(row)
|
|
92
92
|
answers = record['rdata']
|
|
93
|
-
|
|
93
|
+
answers = [record['rdata']] if record['rdata'].class == String
|
|
94
94
|
query = record['rrname'].gsub!(/\.$/,'')
|
|
95
95
|
rrtype = record['rrtype']
|
|
96
96
|
firstseen = Time.at(record['time_first'].to_i)
|
|
97
97
|
lastseen = Time.at(record['time_last'].to_i)
|
|
98
98
|
count = record['count']
|
|
99
99
|
|
|
100
|
-
|
|
100
|
+
answers.each do |answer|
|
|
101
101
|
answer.gsub!(/\.$/,'')
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
0,firstseen,lastseen,count)
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
102
|
+
if record['time_first']
|
|
103
|
+
res << PDNSResult.new(self.class.name,response_time,query,answer,rrtype,
|
|
104
|
+
0,firstseen,lastseen,count, 'yellow')
|
|
105
|
+
else
|
|
106
|
+
res << PDNSResult.new(self.class.name,response_time,query,answer,rrtype)
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
res
|
|
111
|
+
rescue Exception => e
|
|
112
|
+
$stderr.puts "#{self.class.name} Exception: #{e}"
|
|
113
|
+
$stderr.puts page
|
|
114
|
+
raise e
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
117
|
end
|
|
118
|
-
end
|
|
118
|
+
end
|