passivedns-client 2.1.6 → 2.1.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/passivedns/client.rb +68 -36
- data/lib/passivedns/client/cli.rb +139 -139
- data/lib/passivedns/client/provider/bfk.rb +54 -52
- data/lib/passivedns/client/provider/circl.rb +39 -39
- data/lib/passivedns/client/provider/cn360.rb +21 -21
- data/lib/passivedns/client/provider/dnsdb.rb +56 -56
- data/lib/passivedns/client/provider/mnemonic.rb +62 -55
- data/lib/passivedns/client/provider/passivetotal.rb +43 -43
- data/lib/passivedns/client/provider/riskiq.rb +40 -40
- data/lib/passivedns/client/provider/tcpiputils.rb +18 -18
- data/lib/passivedns/client/provider/virustotal.rb +46 -46
- data/lib/passivedns/client/state.rb +236 -236
- data/lib/passivedns/client/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 893747659885f41d56357637504abe671f95f102
|
4
|
+
data.tar.gz: 6f2a91db26389c0286439a6c5b82ca9dbc535a74
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 825f792a8082db19cf426ca8d87c32245ab0e5a8d9d3ee7e6e1442030a2976549affccc2212e71a53339fdb974bd20b5c31b08df3c23423714168753bb126e6d
|
7
|
+
data.tar.gz: b2be17572348b7166bd567009653e5023ee1c0a142f78cb37c355fbe0c6ec38c7c7a89d32015b5f7dd6061ef85a3e152b2b8c62182e9af53c185b8076fd38e10
|
data/lib/passivedns/client.rb
CHANGED
@@ -18,16 +18,48 @@ end
|
|
18
18
|
require 'configparser'
|
19
19
|
|
20
20
|
module PassiveDNS # :nodoc:
|
21
|
+
|
22
|
+
class SecurityControl
|
23
|
+
def allow(user_level)
|
24
|
+
raise "unimplemented"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
class TLPSecurityControl < SecurityControl
|
29
|
+
LEVELS = ['white','green','yellow','red']
|
30
|
+
|
31
|
+
def initialize(tlp)
|
32
|
+
if tlp =~ /(white|green|yellow|red)/i
|
33
|
+
@tlp = tlp.downcase
|
34
|
+
@tlp_level = LEVELS.index(@tlp)
|
35
|
+
else
|
36
|
+
raise "Unknown TLP setting, #{tlp}"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def allow(user_level)
|
41
|
+
user_level = LEVELS.index(user_level.downcase)
|
42
|
+
if user_level == nil
|
43
|
+
raise "Invalid user level, #{user_level}"
|
44
|
+
end
|
45
|
+
return(user_level >= @tlp_level)
|
46
|
+
end
|
47
|
+
|
48
|
+
def to_s()
|
49
|
+
@tlp
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
21
53
|
# struct to contain the results from a PassiveDNS lookup
|
22
|
-
|
54
|
+
class PDNSResult < Struct.new(:source, :response_time, :query, :answer, :rrtype, :ttl, :firstseen, :lastseen, :count, :security); end
|
23
55
|
|
24
56
|
# coodinates the lookups accross all configured PassiveDNS providers
|
25
|
-
|
57
|
+
class Client
|
26
58
|
|
27
59
|
# instantiate and configure all specified PassiveDNS providers
|
28
60
|
# pdns array of passivedns provider names, e.g., ["dnsdb","virustotal"]
|
29
61
|
# configfile filename of the passivedns-client configuration (this should probably be abstracted)
|
30
|
-
|
62
|
+
def initialize(pdns=$passivedns_providers, configfile="#{ENV['HOME']}/.passivedns-client")
|
31
63
|
cp = {}
|
32
64
|
if File.exist?(configfile)
|
33
65
|
cp = ConfigParser.new(configfile)
|
@@ -42,7 +74,7 @@ module PassiveDNS # :nodoc:
|
|
42
74
|
end
|
43
75
|
end
|
44
76
|
|
45
|
-
|
77
|
+
@pdnsdbs = []
|
46
78
|
pdns.uniq.each do |pd|
|
47
79
|
if class_map[pd]
|
48
80
|
@pdnsdbs << class_map[pd].new(cp[pd] || {})
|
@@ -51,38 +83,38 @@ module PassiveDNS # :nodoc:
|
|
51
83
|
end
|
52
84
|
end
|
53
85
|
|
54
|
-
|
55
|
-
|
86
|
+
end #initialize
|
87
|
+
|
56
88
|
# set the debug flag
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
89
|
+
def debug=(d)
|
90
|
+
@pdnsdbs.each do |pdnsdb|
|
91
|
+
pdnsdb.debug = d
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
63
95
|
# perform the query lookup accross all configured PassiveDNS providers
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
96
|
+
def query(item, limit=nil)
|
97
|
+
threads = []
|
98
|
+
@pdnsdbs.each do |pdnsdb|
|
99
|
+
threads << Thread.new(item) do |q|
|
100
|
+
pdnsdb.lookup(q, limit)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
results = []
|
105
|
+
threads.each do |thr|
|
106
|
+
rv = thr.join.value
|
107
|
+
if rv
|
108
|
+
rv.each do |r|
|
109
|
+
if ["A","AAAA","NS","CNAME","PTR"].index(r.rrtype)
|
110
|
+
results << r
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
return results
|
117
|
+
end #query
|
118
|
+
|
119
|
+
end # Client
|
88
120
|
end # PassiveDNS
|
@@ -7,7 +7,7 @@ require 'pp'
|
|
7
7
|
module PassiveDNS # :nodoc:
|
8
8
|
# Handles all the command-line parsing, state tracking, and dispatching queries to the PassiveDNS::Client instance
|
9
9
|
# CLInterface is aliased by CLI
|
10
|
-
|
10
|
+
class CLInterface
|
11
11
|
# generates a mapping between the option letter for each PassiveDNS provider and the class
|
12
12
|
def self.get_letter_map
|
13
13
|
letter_map = {}
|
@@ -41,24 +41,24 @@ module PassiveDNS # :nodoc:
|
|
41
41
|
origARGV = ARGV.dup
|
42
42
|
ARGV.replace(args)
|
43
43
|
opts = GetoptLong.new(
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
44
|
+
[ '--help', '-h', GetoptLong::NO_ARGUMENT ],
|
45
|
+
[ '--debug', '-v', GetoptLong::NO_ARGUMENT ],
|
46
|
+
[ '--database', '-d', GetoptLong::REQUIRED_ARGUMENT ],
|
47
|
+
|
48
|
+
[ '--gdf', '-g', GetoptLong::NO_ARGUMENT ],
|
49
|
+
[ '--graphviz', '-z', GetoptLong::NO_ARGUMENT ],
|
50
|
+
[ '--graphml', '-m', GetoptLong::NO_ARGUMENT ],
|
51
|
+
[ '--csv', '-c', GetoptLong::NO_ARGUMENT ],
|
52
|
+
[ '--xml', '-x', GetoptLong::NO_ARGUMENT ],
|
53
|
+
[ '--yaml', '-y', GetoptLong::NO_ARGUMENT ],
|
54
|
+
[ '--json', '-j', GetoptLong::NO_ARGUMENT ],
|
55
|
+
[ '--text', '-t', GetoptLong::NO_ARGUMENT ],
|
56
|
+
[ '--sep', '-s', GetoptLong::REQUIRED_ARGUMENT ],
|
57
|
+
|
58
|
+
[ '--sqlite3', '-f', GetoptLong::REQUIRED_ARGUMENT ],
|
59
|
+
[ '--recurse', '-r', GetoptLong::REQUIRED_ARGUMENT ],
|
60
|
+
[ '--wait', '-w', GetoptLong::REQUIRED_ARGUMENT ],
|
61
|
+
[ '--limit', '-l', GetoptLong::REQUIRED_ARGUMENT ]
|
62
62
|
)
|
63
63
|
|
64
64
|
letter_map = get_letter_map
|
@@ -78,13 +78,13 @@ module PassiveDNS # :nodoc:
|
|
78
78
|
}
|
79
79
|
|
80
80
|
opts.each do |opt, arg|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
81
|
+
case opt
|
82
|
+
when '--help'
|
83
|
+
options[:help] = true
|
84
|
+
when '--debug'
|
85
|
+
options[:debug] = true
|
86
|
+
when '--database'
|
87
|
+
arg.split(//).each do |c|
|
88
88
|
if c == ','
|
89
89
|
next
|
90
90
|
elsif letter_map[c]
|
@@ -94,60 +94,60 @@ module PassiveDNS # :nodoc:
|
|
94
94
|
usage(letter_map)
|
95
95
|
end
|
96
96
|
end
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
97
|
+
when '--gdf'
|
98
|
+
options[:format] = 'gdf'
|
99
|
+
when '--graphviz'
|
100
|
+
options[:format] = 'graphviz'
|
101
|
+
when '--graphml'
|
102
|
+
options[:format] = 'graphml'
|
103
|
+
when '--csv'
|
104
|
+
options[:format] = 'text'
|
105
|
+
options[:sep] = ','
|
106
|
+
when '--yaml'
|
107
|
+
options[:format] = 'yaml'
|
108
|
+
when '--xml'
|
109
|
+
options[:format] = 'xml'
|
110
|
+
when '--json'
|
111
|
+
options[:format] = 'json'
|
112
|
+
when '--text'
|
113
|
+
options[:format] = 'text'
|
114
|
+
when '--sep'
|
115
|
+
options[:sep] = arg
|
116
|
+
when '--recurse'
|
117
|
+
options[:recursedepth] = arg.to_i
|
118
|
+
when '--wait'
|
119
|
+
options[:wait] = arg.to_i
|
120
|
+
when '--sqlite3'
|
121
|
+
options[:sqlitedb] = arg
|
122
|
+
when '--limit'
|
123
|
+
options[:limit] = arg.to_i
|
124
|
+
else
|
125
|
+
options[:help] = true
|
126
|
+
end
|
127
127
|
end
|
128
128
|
args = ARGV.dup
|
129
129
|
ARGV.replace(origARGV)
|
130
130
|
|
131
131
|
if options[:pdnsdbs].length == 0
|
132
|
-
|
132
|
+
options[:pdnsdbs] << "bfk"
|
133
133
|
end
|
134
134
|
|
135
135
|
if options[:pdnsdbs].index("bfk") and options[:recursedepth] > 1 and options[:wait] < 60
|
136
|
-
|
137
|
-
|
136
|
+
options[:wait] = 60
|
137
|
+
$stderr.puts "Enforcing a minimal 60 second wait when using BFK for recursive crawling"
|
138
138
|
end
|
139
139
|
|
140
140
|
if options[:debug]
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
141
|
+
$stderr.puts "Using the following databases: #{options[:pdnsdbs].join(", ")}"
|
142
|
+
$stderr.puts "Recursions: #{options[:recursedepth]}, Wait time: #{options[:wait]}, Limit: #{options[:limit] or 'none'}"
|
143
|
+
if options[:format] == "text" or options[:format] == "csv"
|
144
|
+
$stderr.puts "Output format: #{options[:format]} (sep=\"#{options[:sep]}\")"
|
145
|
+
else
|
146
|
+
$stderr.puts "Output format: #{options[:format]}"
|
147
|
+
end
|
148
|
+
if ENV['http_proxy']
|
149
|
+
$stderr.puts "Using proxy settings: http_proxy=#{ENV['http_proxy']}, https_proxy=#{ENV['https_proxy']}"
|
150
|
+
end
|
151
151
|
end
|
152
152
|
|
153
153
|
[options, args]
|
@@ -158,36 +158,36 @@ module PassiveDNS # :nodoc:
|
|
158
158
|
def self.usage(letter_map)
|
159
159
|
databases = letter_map.keys.sort.join("")
|
160
160
|
help_text = ""
|
161
|
-
|
161
|
+
help_text << "Usage: #{$0} [-d [#{databases}]] [-g|-v|-m|-c|-x|-y|-j|-t] [-os <sep>] [-f <file>] [-r#|-w#|-v] [-l <count>] <ip|domain|cidr>\n"
|
162
162
|
help_text << "Passive DNS Providers\n"
|
163
|
-
|
163
|
+
help_text << " -d#{databases} uses all of the available passive dns database\n"
|
164
164
|
letter_map.keys.sort.each do |l|
|
165
165
|
help_text << " -d#{l} use #{letter_map[l][0]}\n"
|
166
166
|
end
|
167
|
-
|
168
|
-
|
167
|
+
help_text << " -dvt uses VirusTotal and TCPIPUtils (for example)\n"
|
168
|
+
help_text << "\n"
|
169
169
|
help_text << "Output Formatting\n"
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
170
|
+
help_text << " -g link-nodal GDF visualization definition\n"
|
171
|
+
help_text << " -z link-nodal graphviz visualization definition\n"
|
172
|
+
help_text << " -m link-nodal graphml visualization definition\n"
|
173
|
+
help_text << " -c CSV\n"
|
174
|
+
help_text << " -x XML\n"
|
175
|
+
help_text << " -y YAML\n"
|
176
|
+
help_text << " -j JSON\n"
|
177
|
+
help_text << " -t ASCII text (default)\n"
|
178
|
+
help_text << " -s <sep> specifies a field separator for text output, default is tab\n"
|
179
|
+
help_text << "\n"
|
180
180
|
help_text << "State and Recursion\n"
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
181
|
+
help_text << " -f[file] specifies a sqlite3 database used to read the current state - useful for large result sets and generating graphs of previous runs.\n"
|
182
|
+
help_text << " -r# specifies the levels of recursion to pull. **WARNING** This is quite taxing on the pDNS servers, so use judiciously (never more than 3 or so) or find yourself blocked!\n"
|
183
|
+
help_text << " -w# specifies the amount of time to wait, in seconds, between queries (Default: 0)\n"
|
184
|
+
help_text << " -l <count> limits the number of records returned per passive dns database queried.\n"
|
185
|
+
help_text << "\n"
|
186
186
|
help_text << "Getting Help\n"
|
187
187
|
help_text << " -h hello there. This option produces this helpful help information on how to access help.\n"
|
188
|
-
|
188
|
+
help_text << " -v debugging information\n"
|
189
189
|
|
190
|
-
|
190
|
+
help_text
|
191
191
|
end
|
192
192
|
|
193
193
|
# performs a stateful, recursive (if desired) passive DNS lookup against all specified providers
|
@@ -196,58 +196,58 @@ module PassiveDNS # :nodoc:
|
|
196
196
|
wait = options[:wait]
|
197
197
|
debug = options[:debug]
|
198
198
|
limit = options[:limit]
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
199
|
+
puts "pdnslookup: #{state.level} #{recursedepth}" if debug
|
200
|
+
level = 0
|
201
|
+
while level < recursedepth
|
202
|
+
puts "pdnslookup: #{level} < #{recursedepth}" if debug
|
203
|
+
state.each_query(recursedepth) do |q|
|
204
|
+
rv = pdnsclient.query(q,limit)
|
205
|
+
if rv
|
206
|
+
rv.each do |r|
|
207
|
+
if ["A","AAAA","NS","CNAME","PTR"].index(r.rrtype)
|
208
|
+
puts "pdnslookup: #{r.to_s}" if debug
|
209
|
+
state.add_result(r)
|
210
|
+
end
|
211
|
+
end
|
212
|
+
else
|
213
|
+
state.update_query(rv,'failed')
|
214
|
+
end
|
215
|
+
sleep wait if level < recursedepth
|
216
|
+
end
|
217
|
+
level += 1
|
218
|
+
end
|
219
|
+
state
|
220
220
|
end
|
221
221
|
|
222
222
|
# returns a string transforming all the PassiveDNS::PDNSResult stored in the state object into text/xml/json/etc.
|
223
223
|
def self.results_to_s(state,options)
|
224
224
|
format = options[:format]
|
225
225
|
sep = options[:sep]
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
226
|
+
case format
|
227
|
+
when 'text'
|
228
|
+
PassiveDNS::PDNSResult.members.join(sep)+"\n"+state.to_s(sep)
|
229
|
+
when 'yaml'
|
230
|
+
state.to_yaml
|
231
|
+
when 'xml'
|
232
|
+
state.to_xml
|
233
|
+
when 'json'
|
234
|
+
state.to_json
|
235
|
+
when 'gdf'
|
236
|
+
state.to_gdf
|
237
|
+
when 'graphviz'
|
238
|
+
state.to_graphviz
|
239
|
+
when 'graphml'
|
240
|
+
state.to_graphml
|
241
|
+
end
|
242
242
|
end
|
243
243
|
|
244
244
|
# create a state instance
|
245
245
|
def self.create_state(sqlitedb=nil)
|
246
246
|
state = nil
|
247
247
|
if sqlitedb
|
248
|
-
|
248
|
+
state = PassiveDNS::PDNSToolStateDB.new(sqlitedb)
|
249
249
|
else
|
250
|
-
|
250
|
+
state = PassiveDNS::PDNSToolState.new
|
251
251
|
end
|
252
252
|
end
|
253
253
|
|
@@ -257,10 +257,10 @@ module PassiveDNS # :nodoc:
|
|
257
257
|
if options[:help]
|
258
258
|
return usage(get_letter_map)
|
259
259
|
end
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
260
|
+
if options[:recursedepth] > 3
|
261
|
+
$stderr.puts "WARNING: a recursedepth of > 3 can be abusive, please reconsider: sleeping 60 seconds for sense to come to you (hint: hit CTRL-C)"
|
262
|
+
sleep 60
|
263
|
+
end
|
264
264
|
state = create_state(options[:sqlitedb])
|
265
265
|
state.debug = options[:debug]
|
266
266
|
|
@@ -268,13 +268,13 @@ module PassiveDNS # :nodoc:
|
|
268
268
|
pdnsclient.debug = options[:debug]
|
269
269
|
|
270
270
|
if items.length > 0
|
271
|
-
|
272
|
-
|
273
|
-
|
271
|
+
items.each do |arg|
|
272
|
+
state.add_query(arg,'pending',0)
|
273
|
+
end
|
274
274
|
else
|
275
|
-
|
276
|
-
|
277
|
-
|
275
|
+
$stdin.each_line do |l|
|
276
|
+
state.add_query(l.chomp,'pending',0)
|
277
|
+
end
|
278
278
|
end
|
279
279
|
pdnslookup(state,pdnsclient,options)
|
280
280
|
results_to_s(state,options)
|