passivedns-client 2.1.7 → 2.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/README.md +34 -49
- data/Rakefile +0 -0
- data/lib/passivedns/client.rb +6 -0
- data/lib/passivedns/client/cli.rb +14 -11
- data/lib/passivedns/client/provider/circl.rb +22 -6
- data/lib/passivedns/client/provider/dnsdb.rb +3 -2
- data/lib/passivedns/client/provider/passivetotal.rb +8 -3
- data/lib/passivedns/client/provider/riskiq.rb +22 -6
- data/lib/passivedns/client/provider/virustotal.rb +17 -3
- data/lib/passivedns/client/state.rb +2 -1
- data/lib/passivedns/client/version.rb +1 -1
- data/passivedns-client.gemspec +6 -6
- data/test/test_cli.rb +57 -43
- data/test/test_passivedns-client.rb +73 -166
- metadata +23 -28
- data/lib/passivedns/client/provider/bfk.rb +0 -107
- data/lib/passivedns/client/provider/cn360.rb +0 -111
- data/lib/passivedns/client/provider/mnemonic.rb +0 -111
- data/lib/passivedns/client/provider/tcpiputils.rb +0 -128
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: passivedns-client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.1.
|
4
|
+
version: 2.1.13
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- chrislee35
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-12-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: json
|
@@ -16,84 +16,84 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
26
|
+
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: sqlite3
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
33
|
+
version: '0'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version:
|
40
|
+
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: structformatter
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - "
|
45
|
+
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: 0
|
47
|
+
version: '0'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - "
|
52
|
+
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: 0
|
54
|
+
version: '0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: configparser
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - "
|
59
|
+
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: 0
|
61
|
+
version: '0'
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - "
|
66
|
+
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: 0
|
68
|
+
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: minitest
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- - "
|
73
|
+
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: '
|
75
|
+
version: '0'
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- - "
|
80
|
+
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version: '
|
82
|
+
version: '0'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: bundler
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- - "
|
87
|
+
- - ">="
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: '
|
89
|
+
version: '0'
|
90
90
|
type: :development
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
|
-
- - "
|
94
|
+
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version: '
|
96
|
+
version: '0'
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: rake
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
@@ -127,14 +127,10 @@ files:
|
|
127
127
|
- lib/passivedns/client.rb
|
128
128
|
- lib/passivedns/client/cli.rb
|
129
129
|
- lib/passivedns/client/passivedb.rb
|
130
|
-
- lib/passivedns/client/provider/bfk.rb
|
131
130
|
- lib/passivedns/client/provider/circl.rb
|
132
|
-
- lib/passivedns/client/provider/cn360.rb
|
133
131
|
- lib/passivedns/client/provider/dnsdb.rb
|
134
|
-
- lib/passivedns/client/provider/mnemonic.rb
|
135
132
|
- lib/passivedns/client/provider/passivetotal.rb
|
136
133
|
- lib/passivedns/client/provider/riskiq.rb
|
137
|
-
- lib/passivedns/client/provider/tcpiputils.rb
|
138
134
|
- lib/passivedns/client/provider/virustotal.rb
|
139
135
|
- lib/passivedns/client/state.rb
|
140
136
|
- lib/passivedns/client/version.rb
|
@@ -161,8 +157,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
161
157
|
- !ruby/object:Gem::Version
|
162
158
|
version: '0'
|
163
159
|
requirements: []
|
164
|
-
|
165
|
-
rubygems_version: 2.4.6
|
160
|
+
rubygems_version: 3.1.2
|
166
161
|
signing_key:
|
167
162
|
specification_version: 4
|
168
163
|
summary: Query passive DNS databases
|
@@ -1,107 +0,0 @@
|
|
1
|
-
require 'open-uri'
|
2
|
-
|
3
|
-
module PassiveDNS #:nodoc: don't document this
|
4
|
-
# The Provider module contains all the Passive DNS provider client code
|
5
|
-
module Provider
|
6
|
-
|
7
|
-
# Queries BFK.de's passive DNS database
|
8
|
-
class BFK < PassiveDB
|
9
|
-
# Sets the modules self-reported name to "BFK.de"
|
10
|
-
def self.name
|
11
|
-
"BFK.de"
|
12
|
-
end
|
13
|
-
# Sets the configuration section name to "bfk"
|
14
|
-
def self.config_section_name
|
15
|
-
"bfk"
|
16
|
-
end
|
17
|
-
# Sets the command line database argument to "b"
|
18
|
-
def self.option_letter
|
19
|
-
"b"
|
20
|
-
end
|
21
|
-
|
22
|
-
# :debug enables verbose logging to standard output
|
23
|
-
attr_accessor :debug
|
24
|
-
# === Options
|
25
|
-
# * :debug Sets the debug flag for the module
|
26
|
-
# * "URL" Alternate url for testing. Defaults to "http://www.bfk.de/bfk_dnslogger.html?query="
|
27
|
-
#
|
28
|
-
# === Example Instantiation
|
29
|
-
#
|
30
|
-
# options = {
|
31
|
-
# :debug => true,
|
32
|
-
# "URL" => "http://www.bfk.de/bfk_dnslogger.html?query="
|
33
|
-
# }
|
34
|
-
#
|
35
|
-
# PassiveDNS::Provider::BFK.new(options)
|
36
|
-
#
|
37
|
-
def initialize(options={})
|
38
|
-
@debug = options[:debug] || false
|
39
|
-
@base = options["URL"] || "http://www.bfk.de/bfk_dnslogger.html?query="
|
40
|
-
end
|
41
|
-
|
42
|
-
# Takes a label (either a domain or an IP address) and returns
|
43
|
-
# an array of PassiveDNS::PDNSResult instances with the answers to the query
|
44
|
-
def lookup(label, limit=nil)
|
45
|
-
$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
|
46
|
-
Timeout::timeout(240) {
|
47
|
-
t1 = Time.now
|
48
|
-
open(
|
49
|
-
@base+label,
|
50
|
-
"User-Agent" => "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}"
|
51
|
-
) do |f|
|
52
|
-
t2 = Time.now
|
53
|
-
recs = parse(f.read,t2-t1)
|
54
|
-
if limit
|
55
|
-
recs[0,limit]
|
56
|
-
else
|
57
|
-
recs
|
58
|
-
end
|
59
|
-
end
|
60
|
-
}
|
61
|
-
rescue Timeout::Error => e
|
62
|
-
$stderr.puts "#{self.class.name} lookup timed out: #{label}"
|
63
|
-
end
|
64
|
-
|
65
|
-
private
|
66
|
-
|
67
|
-
# parses the webpage returned by BFK to generate an array of PDNSResult
|
68
|
-
def parse(page,response_time)
|
69
|
-
line = page.unpack('C*').pack('U*').split(/<table/).grep(/ id=\"logger\"/)
|
70
|
-
return [] unless line.length > 0
|
71
|
-
line = line[0].gsub(/[\t\n]/,'').gsub(/<\/table.*/,'')
|
72
|
-
rows = line.split(/<tr.*?>/)
|
73
|
-
res = []
|
74
|
-
rows.collect do |row|
|
75
|
-
r = row.split(/<td>/).map{|x| x.gsub(/<.*?>/,'').gsub(/\&.*?;/,'')}[1,1000]
|
76
|
-
if r and r[0] =~ /\w/
|
77
|
-
# BFK includes the MX weight in the answer response. First, find the MX records,
|
78
|
-
# then dump the weight to present a consistent record name to the collecting
|
79
|
-
# array. Otherwise the other repositories will present the same answer and
|
80
|
-
# your results will become cluttered with duplicates.
|
81
|
-
if r[1] == "MX" then
|
82
|
-
# MX lines look like "5 mx.domain.tld", so split on the space and assign r[2] (:answer) to the latter part.
|
83
|
-
#s = r[2].split(/\w/).map{|x| x}[1,1000]
|
84
|
-
# r[2] = s[1]
|
85
|
-
r[2] =~ /[0-9]+?\s(.+)/
|
86
|
-
s=$1
|
87
|
-
#puts "DEBUG: == BFK: MX Parsing Strip: Answer: " + r[2] + " : mod: " + s if @debug
|
88
|
-
r[2] = s
|
89
|
-
|
90
|
-
######### FIX BLANKS FOR MX
|
91
|
-
|
92
|
-
end
|
93
|
-
query = r[0]
|
94
|
-
answer = r[2]
|
95
|
-
rrtype = r[1]
|
96
|
-
res << PDNSResult.new(self.class.name,response_time,query,answer,rrtype,'white')
|
97
|
-
end
|
98
|
-
end
|
99
|
-
res
|
100
|
-
rescue Exception => e
|
101
|
-
$stderr.puts "#{self.class.name} Exception: #{e}"
|
102
|
-
raise e
|
103
|
-
end
|
104
|
-
|
105
|
-
end
|
106
|
-
end
|
107
|
-
end
|
@@ -1,111 +0,0 @@
|
|
1
|
-
require 'net/http'
|
2
|
-
require 'net/https'
|
3
|
-
require 'openssl'
|
4
|
-
require 'json'
|
5
|
-
require 'digest/md5'
|
6
|
-
|
7
|
-
module PassiveDNS #:nodoc: don't document this
|
8
|
-
# The Provider module contains all the Passive DNS provider client code
|
9
|
-
module Provider
|
10
|
-
# Queries 360.cn's passive DNS database
|
11
|
-
class CN360 < PassiveDB
|
12
|
-
# Sets the modules self-reported name to "360.cn"
|
13
|
-
def self.name
|
14
|
-
"360.cn"
|
15
|
-
end
|
16
|
-
# Sets the configuration section name to "cn360"
|
17
|
-
def self.config_section_name
|
18
|
-
"cn360"
|
19
|
-
end
|
20
|
-
# Sets the command line database argument to "3"
|
21
|
-
def self.option_letter
|
22
|
-
"3"
|
23
|
-
end
|
24
|
-
|
25
|
-
# :debug enables verbose logging to standard output
|
26
|
-
attr_accessor :debug
|
27
|
-
# === Options
|
28
|
-
# * :debug Sets the debug flag for the module
|
29
|
-
# * "API" REQUIRED: http://some.web.address.for.their.api
|
30
|
-
# * "API_ID" REQUIRED: a username that is given when you register
|
31
|
-
# * "API_KEY" REQUIRED: a long and random password of sorts that is used along with the page request to generate a per page API key
|
32
|
-
#
|
33
|
-
# === Example Instantiation
|
34
|
-
#
|
35
|
-
# options = {
|
36
|
-
# :debug => true,
|
37
|
-
# "API" => "http://some.web.address.for.their.api",
|
38
|
-
# "API_ID" => "360user",
|
39
|
-
# "API_KEY" => "360apikey"
|
40
|
-
# }
|
41
|
-
#
|
42
|
-
# PassiveDNS::Provider::CN360.new(options)
|
43
|
-
#
|
44
|
-
def initialize(options={})
|
45
|
-
@debug = options[:debug] || false
|
46
|
-
["API", "API_ID", "API_KEY"].each do |opt|
|
47
|
-
if not options[opt]
|
48
|
-
raise "Field #{opt} is required. See README.md"
|
49
|
-
end
|
50
|
-
end
|
51
|
-
@cp = options
|
52
|
-
end
|
53
|
-
|
54
|
-
# Takes a label (either a domain or an IP address) and returns
|
55
|
-
# an array of PassiveDNS::PDNSResult instances with the answers to the query
|
56
|
-
def lookup(label, limit=10000)
|
57
|
-
table = "rrset"
|
58
|
-
if label =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/ or label =~ /^[0-9a-fA-F]+:[0-9a-fA-F:]+[0-9a-fA-F]$/
|
59
|
-
table = "rdata"
|
60
|
-
end
|
61
|
-
limit ||= 10000
|
62
|
-
path = "/api/#{table}/keyword/#{label}/count/#{limit}/"
|
63
|
-
url = @cp["API"]+path
|
64
|
-
url = URI.parse url
|
65
|
-
http = Net::HTTP.new(url.host, url.port)
|
66
|
-
http.use_ssl = (url.scheme == 'https')
|
67
|
-
http.verify_mode = OpenSSL::SSL::VERIFY_NONE # I hate doing this
|
68
|
-
http.verify_depth = 5
|
69
|
-
request = Net::HTTP::Get.new(url.path)
|
70
|
-
request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
|
71
|
-
request.add_field('Accept', 'application/json')
|
72
|
-
request.add_field("X-BashTokid", @cp["API_ID"])
|
73
|
-
token = Digest::MD5.hexdigest(path+@cp["API_KEY"])
|
74
|
-
$stderr.puts "DEBUG: cn360 url = #{url} token = #{token}" if @debug
|
75
|
-
request.add_field("X-BashToken", token)
|
76
|
-
t1 = Time.now
|
77
|
-
response = http.request(request)
|
78
|
-
t2 = Time.now
|
79
|
-
recs = parse_json(response.body, label, t2-t1)
|
80
|
-
if limit
|
81
|
-
recs[0,limit]
|
82
|
-
else
|
83
|
-
recs
|
84
|
-
end
|
85
|
-
end
|
86
|
-
|
87
|
-
private
|
88
|
-
|
89
|
-
# parses the response of 360.cn's JSON reply to generate an array of PDNSResult
|
90
|
-
def parse_json(page,query,response_time=0)
|
91
|
-
res = []
|
92
|
-
data = JSON.parse(page)
|
93
|
-
data.each do |row|
|
94
|
-
time_first = (row["time_first"]) ? Time.at(row["time_first"].to_i) : nil
|
95
|
-
time_last = (row["time_last"]) ? Time.at(row["time_last"].to_i) : nil
|
96
|
-
count = row["count"] || 0
|
97
|
-
query = row["rrname"]
|
98
|
-
answers = row["rdata"].gsub(/;$/,'').split(/;/)
|
99
|
-
rrtype = row["rrtype"]
|
100
|
-
answers.each do |answer|
|
101
|
-
res << PDNSResult.new(self.class.name, response_time, query, answer, rrtype, time_first, time_last, count, 'yellow')
|
102
|
-
end
|
103
|
-
end
|
104
|
-
res
|
105
|
-
rescue Exception => e
|
106
|
-
$stderr.puts "#{self.class.name} Exception: #{e}"
|
107
|
-
raise e
|
108
|
-
end
|
109
|
-
end
|
110
|
-
end
|
111
|
-
end
|
@@ -1,111 +0,0 @@
|
|
1
|
-
# DESCRIPTION: Module to query Mnemonic's passive DNS repository
|
2
|
-
# CONTRIBUTOR: Drew Hunt (pinowudi@yahoo.com)
|
3
|
-
require 'net/http'
|
4
|
-
require 'net/https'
|
5
|
-
require 'openssl'
|
6
|
-
|
7
|
-
module PassiveDNS #:nodoc: don't document this
|
8
|
-
# The Provider module contains all the Passive DNS provider client code
|
9
|
-
module Provider
|
10
|
-
# Queries Mnemonic's passive DNS database
|
11
|
-
class Mnemonic < PassiveDB
|
12
|
-
# Sets the modules self-reported name to "Mnemonic"
|
13
|
-
def self.name
|
14
|
-
"Mnemonic"
|
15
|
-
end
|
16
|
-
# Sets the configuration section name to "mnemonic"
|
17
|
-
def self.config_section_name
|
18
|
-
"mnemonic"
|
19
|
-
end
|
20
|
-
# Sets the command line database argument to "m"
|
21
|
-
def self.option_letter
|
22
|
-
"m"
|
23
|
-
end
|
24
|
-
|
25
|
-
# :debug enables verbose logging to standard output
|
26
|
-
attr_accessor :debug
|
27
|
-
# === Options
|
28
|
-
# * :debug Sets the debug flag for the module
|
29
|
-
# * "APIKEY" The API key associated with Mnemonic for doing automated queries
|
30
|
-
# * "URL" Alternate url for testing. Defaults to "https://api.mnemonic.no/pdns/v3/"
|
31
|
-
#
|
32
|
-
# === Example Instantiation
|
33
|
-
#
|
34
|
-
# options = {
|
35
|
-
# :debug => true,
|
36
|
-
# "APIKEY" => "01234567890abcdef01234567890abcdef012345",
|
37
|
-
# "URL" => "https://api.mnemonic.no/pdns/v3/"
|
38
|
-
# }
|
39
|
-
#
|
40
|
-
# PassiveDNS::Provider::Mnemonic.new(options)
|
41
|
-
#
|
42
|
-
def initialize(options={})
|
43
|
-
@debug = options[:debug] || false
|
44
|
-
@apikey = options["APIKEY"]
|
45
|
-
@url = options["URL"] || "https://api.mnemonic.no/pdns/v3/"
|
46
|
-
if @url == "https://passivedns.mnemonic.no/api1/?apikey="
|
47
|
-
@url = "https://api.mnemonic.no/pdns/v3/"
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
# Takes a label (either a domain or an IP address) and returns
|
52
|
-
# an array of PassiveDNS::PDNSResult instances with the answers to the query
|
53
|
-
def lookup(label, limit=nil)
|
54
|
-
$stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
|
55
|
-
Timeout::timeout(240) {
|
56
|
-
url = "#{@url}#{label}"
|
57
|
-
$stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
|
58
|
-
url = URI.parse url
|
59
|
-
http = Net::HTTP.new(url.host, url.port)
|
60
|
-
http.use_ssl = (url.scheme == 'https')
|
61
|
-
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
62
|
-
http.verify_depth = 5
|
63
|
-
request = Net::HTTP::Get.new(url.path)
|
64
|
-
request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
|
65
|
-
if @apikey
|
66
|
-
request.add_field("Argus-API-Key", @apikey)
|
67
|
-
end
|
68
|
-
t1 = Time.now
|
69
|
-
response = http.request(request)
|
70
|
-
t2 = Time.now
|
71
|
-
recs = parse_json(response.body, label, t2-t1)
|
72
|
-
if limit
|
73
|
-
recs[0,limit]
|
74
|
-
else
|
75
|
-
recs
|
76
|
-
end
|
77
|
-
}
|
78
|
-
rescue Timeout::Error => e
|
79
|
-
$stderr.puts "#{self.class.name} lookup timed out: #{label}"
|
80
|
-
end
|
81
|
-
|
82
|
-
private
|
83
|
-
|
84
|
-
# parses the response of mnemonic's JSON reply to generate an array of PDNSResult
|
85
|
-
def parse_json(page,query,response_time=0)
|
86
|
-
res = []
|
87
|
-
data = JSON.parse(page)
|
88
|
-
if data['data']
|
89
|
-
data['data'].each do |row|
|
90
|
-
if row['query']
|
91
|
-
query = row['query']
|
92
|
-
answer = row['answer']
|
93
|
-
rrtype = row['rrtype'].upcase
|
94
|
-
ttl = row['maxTtl'].to_i
|
95
|
-
firstseen = Time.at(row['firstSeenTimestamp'].to_i / 1000)
|
96
|
-
lastseen = Time.at(row['lastSeenTimestamp'].to_i / 1000)
|
97
|
-
tlp = row['tlp']
|
98
|
-
r = PDNSResult.new(self.class.name,response_time, query, answer, rrtype, ttl, firstseen, lastseen, tlp)
|
99
|
-
res << r
|
100
|
-
end
|
101
|
-
end
|
102
|
-
end
|
103
|
-
res
|
104
|
-
rescue Exception => e
|
105
|
-
$stderr.puts "#{self.class.name} Exception: #{e}"
|
106
|
-
raise e
|
107
|
-
end
|
108
|
-
|
109
|
-
end
|
110
|
-
end
|
111
|
-
end
|