passivedns-client 2.1.6 → 2.1.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +0 -0
  3. data/Gemfile +0 -0
  4. data/LICENSE.txt +0 -0
  5. data/README.md +3 -0
  6. data/lib/passivedns/client.rb +77 -37
  7. data/lib/passivedns/client/cli.rb +149 -141
  8. data/lib/passivedns/client/passivedb.rb +0 -0
  9. data/lib/passivedns/client/provider/bfk.rb +56 -52
  10. data/lib/passivedns/client/provider/circl.rb +55 -39
  11. data/lib/passivedns/client/provider/cn360.rb +44 -33
  12. data/lib/passivedns/client/provider/dnsdb.rb +57 -56
  13. data/lib/passivedns/client/provider/mnemonic.rb +63 -55
  14. data/lib/passivedns/client/provider/passivetotal.rb +48 -43
  15. data/lib/passivedns/client/provider/riskiq.rb +59 -43
  16. data/lib/passivedns/client/provider/tcpiputils.rb +20 -19
  17. data/lib/passivedns/client/provider/virustotal.rb +60 -46
  18. data/lib/passivedns/client/state.rb +237 -236
  19. data/lib/passivedns/client/version.rb +1 -1
  20. data/passivedns-client.gemspec +6 -6
  21. data/test/helper.rb +0 -0
  22. data/test/test_cli.rb +34 -5
  23. data/test/test_passivedns-client.rb +153 -146
  24. metadata +23 -24
data/lib/passivedns/client/provider/mnemonic.rb CHANGED
@@ -8,7 +8,7 @@ module PassiveDNS #:nodoc: don't document this
8
8
  # The Provider module contains all the Passive DNS provider client code
9
9
  module Provider
10
10
  # Queries Mnemonic's passive DNS database
11
- class Mnemonic < PassiveDB
11
+ class Mnemonic < PassiveDB
12
12
  # Sets the modules self-reported name to "Mnemonic"
13
13
  def self.name
14
14
  "Mnemonic"
@@ -26,79 +26,87 @@ module PassiveDNS #:nodoc: don't document this
26
26
  attr_accessor :debug
27
27
  # === Options
28
28
  # * :debug Sets the debug flag for the module
29
- # * "APIKEY" REQUIRED: The API key associated with Mnemonic
30
- # * "URL" Alternate url for testing. Defaults to "https://passivedns.mnemonic.no/api1/?apikey="
29
+ # * "APIKEY" The API key associated with Mnemonic for doing automated queries
30
+ # * "URL" Alternate url for testing. Defaults to "https://api.mnemonic.no/pdns/v3/"
31
31
  #
32
32
  # === Example Instantiation
33
33
  #
34
34
  # options = {
35
35
  # :debug => true,
36
36
  # "APIKEY" => "01234567890abcdef01234567890abcdef012345",
37
- # "URL" => "https://passivedns.mnemonic.no/api1/?apikey="
37
+ # "URL" => "https://api.mnemonic.no/pdns/v3/"
38
38
  # }
39
39
  #
40
40
  # PassiveDNS::Provider::Mnemonic.new(options)
41
41
  #
42
- def initialize(options={})
42
+ def initialize(options={})
43
43
  @debug = options[:debug] || false
44
- @apikey = options["APIKEY"] || raise("#{self.class.name} requires an APIKEY")
45
- @url = options["URL"] || "https://passivedns.mnemonic.no/api1/?apikey="
46
- end
44
+ @timeout = options[:timeout] || 20
45
+ @apikey = options["APIKEY"]
46
+ @url = options["URL"] || "https://api.mnemonic.no/pdns/v3/"
47
+ if @url == "https://passivedns.mnemonic.no/api1/?apikey="
48
+ @url = "https://api.mnemonic.no/pdns/v3/"
49
+ end
50
+ end
47
51
 
48
52
  # Takes a label (either a domain or an IP address) and returns
49
53
  # an array of PassiveDNS::PDNSResult instances with the answers to the query
50
- def lookup(label, limit=nil)
51
- $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
52
- Timeout::timeout(240) {
53
- url = "#{@url}#{@apikey}&query=#{label}&method=exact"
54
- $stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
55
- url = URI.parse url
56
- http = Net::HTTP.new(url.host, url.port)
57
- http.use_ssl = (url.scheme == 'https')
58
- http.verify_mode = OpenSSL::SSL::VERIFY_NONE
59
- http.verify_depth = 5
60
- request = Net::HTTP::Get.new(url.path+"?"+url.query)
61
- request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
62
- t1 = Time.now
63
- response = http.request(request)
64
- t2 = Time.now
65
- recs = parse_json(response.body, label, t2-t1)
66
- if limit
67
- recs[0,limit]
68
- else
69
- recs
70
- end
71
- }
72
- rescue Timeout::Error => e
73
- $stderr.puts "#{self.class.name} lookup timed out: #{label}"
74
- end
54
+ def lookup(label, limit=nil)
55
+ $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
56
+ Timeout::timeout(@timeout) {
57
+ url = "#{@url}#{label}"
58
+ $stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
59
+ url = URI.parse url
60
+ http = Net::HTTP.new(url.host, url.port)
61
+ http.use_ssl = (url.scheme == 'https')
62
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
63
+ http.verify_depth = 5
64
+ request = Net::HTTP::Get.new(url.path)
65
+ request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
66
+ if @apikey
67
+ request.add_field("Argus-API-Key", @apikey)
68
+ end
69
+ t1 = Time.now
70
+ response = http.request(request)
71
+ t2 = Time.now
72
+ recs = parse_json(response.body, label, t2-t1)
73
+ if limit
74
+ recs[0,limit]
75
+ else
76
+ recs
77
+ end
78
+ }
79
+ rescue Timeout::Error
80
+ $stderr.puts "#{self.class.name} lookup timed out: #{label}"
81
+ end
75
82
 
76
83
  private
77
84
 
78
85
  # parses the response of mnemonic's JSON reply to generate an array of PDNSResult
79
- def parse_json(page,query,response_time=0)
80
- res = []
81
- data = JSON.parse(page)
82
- if data['result']
83
- data['result'].each do |row|
84
- if row['query']
86
+ def parse_json(page,query,response_time=0)
87
+ res = []
88
+ data = JSON.parse(page)
89
+ if data['data']
90
+ data['data'].each do |row|
91
+ if row['query']
85
92
  query = row['query']
86
93
  answer = row['answer']
87
- rrtype = row['type'].upcase
88
- tty = row['ttl'].to_i
89
- firstseen = Time.at(row['first'].to_i)
90
- lastseen = Time.at(row['last'].to_i)
91
- res << PDNSResult.new(self.class.name,response_time,
92
- query, answer, rrtype, ttl, firstseen, lastseen)
93
- end
94
- end
95
- end
96
- res
97
- rescue Exception => e
98
- $stderr.puts "#{self.class.name} Exception: #{e}"
99
- raise e
100
- end
94
+ rrtype = row['rrtype'].upcase
95
+ ttl = row['maxTtl'].to_i
96
+ firstseen = Time.at(row['firstSeenTimestamp'].to_i / 1000)
97
+ lastseen = Time.at(row['lastSeenTimestamp'].to_i / 1000)
98
+ tlp = row['tlp']
99
+ r = PDNSResult.new(self.class.name,response_time, query, answer, rrtype, ttl, firstseen, lastseen, tlp)
100
+ res << r
101
+ end
102
+ end
103
+ end
104
+ res
105
+ rescue Exception => e
106
+ $stderr.puts "#{self.class.name} Exception: #{e}"
107
+ raise e
108
+ end
101
109
 
102
- end
110
+ end
103
111
  end
104
- end
112
+ end
data/lib/passivedns/client/provider/passivetotal.rb CHANGED
@@ -9,7 +9,7 @@ module PassiveDNS #:nodoc: don't document this
9
9
  # The Provider module contains all the Passive DNS provider client code
10
10
  module Provider
11
11
  # Queries PassiveTotal's passive DNS database
12
- class PassiveTotal < PassiveDB
12
+ class PassiveTotal < PassiveDB
13
13
  # Sets the modules self-reported name to "PassiveTotal"
14
14
  def self.name
15
15
  "PassiveTotal"
@@ -52,65 +52,70 @@ module PassiveDNS #:nodoc: don't document this
52
52
  #
53
53
  # PassiveDNS::Provider::PassiveTotal.new(options)
54
54
  #
55
- def initialize(options={})
55
+ def initialize(options={})
56
56
  @debug = options[:debug] || false
57
+ @timeout = options[:timeout] || 20
57
58
  @username = options["USERNAME"] || raise("#{self.class.name} requires a USERNAME")
58
59
  @apikey = options["APIKEY"] || raise("#{self.class.name} requires an APIKEY")
59
60
  @url = options["URL"] || "https://api.passivetotal.org/v2/dns/passive"
60
- end
61
+ end
61
62
 
62
63
  # Takes a label (either a domain or an IP address) and returns
63
64
  # an array of PassiveDNS::PDNSResult instances with the answers to the query
64
- def lookup(label, limit=nil)
65
- $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
66
- Timeout::timeout(240) {
67
- url = @url+"?query=#{label}"
68
- $stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
69
- url = URI.parse url
70
- http = Net::HTTP.new(url.host, url.port)
71
- http.use_ssl = (url.scheme == 'https')
72
- http.verify_mode = OpenSSL::SSL::VERIFY_NONE
73
- http.verify_depth = 5
74
- request = Net::HTTP::Get.new(url.request_uri)
65
+ def lookup(label, limit=nil)
66
+ $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
67
+ Timeout::timeout(@timeout) {
68
+ url = @url+"?query=#{label}"
69
+ $stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
70
+ url = URI.parse url
71
+ http = Net::HTTP.new(url.host, url.port)
72
+ http.use_ssl = (url.scheme == 'https')
73
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
74
+ http.verify_depth = 5
75
+ request = Net::HTTP::Get.new(url.request_uri)
75
76
  request.basic_auth(@username, @apikey)
76
- request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
77
+ request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
77
78
  #request.set_form_data({"api_key" => @apikey, "query" => label})
78
- t1 = Time.now
79
- response = http.request(request)
80
- t2 = Time.now
81
- recs = parse_json(response.body, label, t2-t1)
82
- if limit
83
- recs[0,limit]
84
- else
85
- recs
86
- end
87
- }
88
- rescue Timeout::Error => e
89
- $stderr.puts "#{self.class.name} lookup timed out: #{label}"
90
- end
79
+ t1 = Time.now
80
+ response = http.request(request)
81
+ t2 = Time.now
82
+ recs = parse_json(response.body, label, t2-t1)
83
+ if limit
84
+ recs[0,limit]
85
+ else
86
+ recs
87
+ end
88
+ }
89
+ rescue Timeout::Error
90
+ $stderr.puts "#{self.class.name} lookup timed out: #{label}"
91
+ end
91
92
 
92
93
  private
93
94
 
94
95
  # parses the response of passivetotals's JSON reply to generate an array of PDNSResult
95
- def parse_json(page,query,response_time=0)
96
- res = []
97
- data = JSON.parse(page)
96
+ def parse_json(page,query,response_time=0)
97
+ res = []
98
+ data = JSON.parse(page)
99
+ pp data
100
+ if data['message']
101
+ raise "#{self.class.name} Error: #{data['message']}"
102
+ end
98
103
  query = data['queryValue']
99
- if data['results']
100
- data['results'].each do |row|
104
+ if data['results']
105
+ data['results'].each do |row|
101
106
  first_seen = (row['firstSeen'] == "None") ? nil : Time.parse(row['firstSeen']+" +0000")
102
107
  last_seen = (row['lastSeen'] == "None") ? nil : Time.parse(row['lastSeen']+" +0000")
103
108
  value = row['resolve']
104
109
  source = row['source'].join(",")
105
- res << PDNSResult.new(self.class.name+"/"+source,response_time,
106
- query, value, "A", 0, first_seen, last_seen)
107
- end
108
- end
109
- res
110
- rescue Exception => e
111
- $stderr.puts "#{self.class.name} Exception: #{e}"
112
- raise e
113
- end
114
- end
110
+ res << PDNSResult.new(self.class.name+"/"+source,response_time,
111
+ query, value, "A", 0, first_seen, last_seen, 'yellow')
112
+ end
113
+ end
114
+ res
115
+ rescue Exception => e
116
+ $stderr.puts "#{self.class.name} Exception: #{e}"
117
+ raise e
118
+ end
119
+ end
115
120
  end
116
121
  end
data/lib/passivedns/client/provider/riskiq.rb CHANGED
@@ -9,7 +9,7 @@ module PassiveDNS #:nodoc: don't document this
9
9
  # The Provider module contains all the Passive DNS provider client code
10
10
  module Provider
11
11
  # Queries RiskIQ's passive DNS database
12
- class RiskIQ < PassiveDB
12
+ class RiskIQ < PassiveDB
13
13
  # Sets the modules self-reported name to "RiskIQ"
14
14
  def self.name
15
15
  "RiskIQ"
@@ -44,20 +44,21 @@ module PassiveDNS #:nodoc: don't document this
44
44
  #
45
45
  # PassiveDNS::Provider::RiskIQ.new(options)
46
46
  #
47
- def initialize(options={})
47
+ def initialize(options={})
48
48
  @debug = options[:debug] || false
49
+ @timeout = options[:timeout] || 20
49
50
  @token = options["API_TOKEN"] || raise("#{self.class.name} requires an API_TOKEN")
50
51
  @privkey = options["API_PRIVATE_KEY"] || raise("#{self.class.name} requires an API_PRIVATE_KEY")
51
- @server = options["API_SERVER"] || "ws.riskiq.net"
52
52
  @version = options["API_VERSION"] || "v1"
53
+ @server = options["API_SERVER"] || api_settings[@version][:server]
53
54
  @url = "https://#{@server}/#{@version}"
54
- end
55
+ end
55
56
 
56
57
  # Takes a label (either a domain or an IP address) and returns
57
58
  # an array of PassiveDNS::PDNSResult instances with the answers to the query
58
- def lookup(label, limit=nil)
59
- $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
60
- Timeout::timeout(240) {
59
+ def lookup(label, limit=nil)
60
+ $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
61
+ Timeout::timeout(@timeout) {
61
62
  url = nil
62
63
  params = {"rrType" => "", "maxResults" => limit || 1000}
63
64
 
@@ -65,8 +66,10 @@ module PassiveDNS #:nodoc: don't document this
65
66
  url = @url+"/dns/data"
66
67
  params["ip"] = label
67
68
  else
68
- url = @url+"/dns/name"
69
- params["name"] = label
69
+ resource = api_settings[@version][:resource]
70
+ param = api_settings[@version][:param]
71
+ url = @url+"/dns/#{resource}"
72
+ params[param] = label
70
73
  end
71
74
  url << "?"
72
75
  params.each do |k,v|
@@ -74,38 +77,51 @@ module PassiveDNS #:nodoc: don't document this
74
77
  end
75
78
  url.gsub!(/\&$/,"")
76
79
 
77
- $stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
78
- url = URI.parse url
79
- http = Net::HTTP.new(url.host, url.port)
80
- http.use_ssl = (url.scheme == 'https')
81
- http.verify_mode = OpenSSL::SSL::VERIFY_NONE
82
- http.verify_depth = 5
83
- request = Net::HTTP::Get.new(url.request_uri)
84
- request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
80
+ $stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
81
+ url = URI.parse url
82
+ http = Net::HTTP.new(url.host, url.port)
83
+ http.use_ssl = (url.scheme == 'https')
84
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
85
+ http.verify_depth = 5
86
+ request = Net::HTTP::Get.new(url.request_uri)
87
+ request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
85
88
  request.add_field('Accept', 'Application/JSON')
86
89
  request.add_field('Content-Type', 'Application/JSON')
87
90
  request.basic_auth(@token, @privkey)
88
- t1 = Time.now
89
- response = http.request(request)
90
- t2 = Time.now
91
- recs = parse_json(response.body, label, t2-t1)
92
- if limit
93
- recs[0,limit]
94
- else
95
- recs
96
- end
97
- }
98
- rescue Timeout::Error => e
99
- $stderr.puts "#{self.class.name} lookup timed out: #{label}"
100
- end
91
+ t1 = Time.now
92
+ response = http.request(request)
93
+ t2 = Time.now
94
+ recs = parse_json(response.body, label, t2-t1)
95
+ if limit
96
+ recs[0,limit]
97
+ else
98
+ recs
99
+ end
100
+ }
101
+ rescue Timeout::Error
102
+ $stderr.puts "#{self.class.name} lookup timed out: #{label}"
103
+ end
101
104
 
102
105
  private
103
106
 
107
+ def api_settings
108
+ @api_settings ||= {
109
+ 'v1' => { server: "ws.riskiq.net", resource: 'name', param: 'name' },
110
+ 'v2' => { server: "api.passivetotal.org", resource: 'passive', param: 'query' }
111
+ }
112
+ end
113
+
104
114
  # parses the response of riskiq's JSON reply to generate an array of PDNSResult
105
- def parse_json(page,query,response_time=0)
106
- res = []
107
- data = JSON.parse(page)
108
- if data['records']
115
+ def parse_json(page,query,response_time=0)
116
+ res = []
117
+ data = JSON.parse(page)
118
+ if data['message']
119
+ if data['message'] =~ /quota_exceeded/
120
+ $stderr.puts "ERROR: quota exceeded."
121
+ return res
122
+ end
123
+ end
124
+ if data['records']
109
125
  data['records'].each do |record|
110
126
  name = record['name'].gsub!(/\.$/,'')
111
127
  type = record['rrtype']
@@ -114,17 +130,17 @@ module PassiveDNS #:nodoc: don't document this
114
130
  count = record['count']
115
131
  record['data'].each do |datum|
116
132
  datum.gsub!(/\.$/,'')
117
- res << PDNSResult.new(self.class.name,response_time,
118
- name, datum, type, 0, first_seen, last_seen, count)
133
+ res << PDNSResult.new(self.class.name,response_time,
134
+ name, datum, type, 0, first_seen, last_seen, count, 'yellow')
119
135
  end
120
136
  end
121
137
  end
122
- res
123
- rescue Exception => e
124
- $stderr.puts "#{self.class.name} Exception: #{e}"
125
- raise e
126
- end
138
+ res
139
+ rescue Exception => e
140
+ $stderr.puts "#{self.class.name} Exception: #{e}"
141
+ raise e
142
+ end
127
143
 
128
- end
144
+ end
129
145
  end
130
- end
146
+ end
data/lib/passivedns/client/provider/tcpiputils.rb CHANGED
@@ -42,6 +42,7 @@ module PassiveDNS #:nodoc: don't document this
42
42
  #
43
43
  def initialize(options={})
44
44
  @debug = options[:debug] || false
45
+ @timeout = options[:timeout] || 20
45
46
  @apikey = options["APIKEY"] || raise("#{self.class.name} requires an APIKEY. See README.md")
46
47
  @url = options["URL"] || "https://www.utlsapi.com/api.php?version=1.0&apikey="
47
48
  end
@@ -53,18 +54,18 @@ module PassiveDNS #:nodoc: don't document this
53
54
  type = (label.match(/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/)) ? "domainneighbors" : "domainipdnshistory"
54
55
  url = "#{@url}#{@apikey}&type=#{type}&q=#{label}"
55
56
  recs = []
56
- Timeout::timeout(240) {
57
- url = URI.parse url
58
- http = Net::HTTP.new(url.host, url.port)
59
- http.use_ssl = (url.scheme == 'https')
60
- http.verify_mode = OpenSSL::SSL::VERIFY_NONE
61
- http.verify_depth = 5
62
- request = Net::HTTP::Get.new(url.path+"?"+url.query)
63
- request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
64
- t1 = Time.now
65
- response = http.request(request)
66
- delta = (Time.now - t1).to_f
67
- reply = JSON.parse(response.body)
57
+ Timeout::timeout(@timeout) {
58
+ url = URI.parse url
59
+ http = Net::HTTP.new(url.host, url.port)
60
+ http.use_ssl = (url.scheme == 'https')
61
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
62
+ http.verify_depth = 5
63
+ request = Net::HTTP::Get.new(url.path+"?"+url.query)
64
+ request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
65
+ t1 = Time.now
66
+ response = http.request(request)
67
+ delta = (Time.now - t1).to_f
68
+ reply = JSON.parse(response.body)
68
69
  if reply["status"] and reply["status"] == "succeed"
69
70
  question = reply["data"]["question"]
70
71
  recs = format_recs(reply["data"], question, delta)
@@ -76,10 +77,10 @@ module PassiveDNS #:nodoc: don't document this
76
77
  else
77
78
  recs
78
79
  end
79
- }
80
- rescue Timeout::Error => e
81
- $stderr.puts "#{self.class.name} lookup timed out: #{label}"
82
- end
80
+ }
81
+ rescue Timeout::Error
82
+ $stderr.puts "#{self.class.name} lookup timed out: #{label}"
83
+ end
83
84
 
84
85
  private
85
86
 
@@ -110,13 +111,13 @@ module PassiveDNS #:nodoc: don't document this
110
111
  when "domains"
111
112
  data.each do |rec|
112
113
  lastseen = (rec["updatedate"]) ? Date.parse(rec["updatedate"]) : nil
113
- recs << PDNSResult.new(self.class.name, delta, rec, question, "A", nil, nil, nil, nil)
114
+ recs << PDNSResult.new(self.class.name, delta, rec, question, "A", nil, nil, lastseen, nil, 'yellow')
114
115
  end
115
116
  end
116
117
  if add_records
117
118
  data.each do |rec|
118
119
  lastseen = (rec["updatedate"]) ? Date.parse(rec["updatedate"]) : nil
119
- recs << PDNSResult.new(self.class.name, delta, question, rec[fieldname], rrtype, nil, nil, lastseen, nil)
120
+ recs << PDNSResult.new(self.class.name, delta, question, rec[fieldname], rrtype, nil, nil, lastseen, nil, 'yellow')
120
121
  end
121
122
  end
122
123
  end
@@ -125,4 +126,4 @@ module PassiveDNS #:nodoc: don't document this
125
126
 
126
127
  end
127
128
  end
128
- end
129
+ end