domain-probe 1.2.0 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/domain-probe.rb +77 -162
- data/lib/domain-probe/crawling_policy.rb +0 -2
- data/lib/domain-probe/guessing_policy.rb +3 -1
- data/lib/domain-probe/resolver_patch.rb +41 -2
- data/lib/domain-probe/thread_executor.rb +168 -0
- data/lib/domain-probe/util.rb +15 -17
- data/lib/domain-probe/version.rb +1 -1
- metadata +13 -12
data/lib/domain-probe.rb
CHANGED
@@ -1,15 +1,15 @@
|
|
1
|
+
require 'domain-probe/tld'
|
1
2
|
require 'domain-probe/resolver_patch'
|
2
3
|
require 'domain-probe/util'
|
3
4
|
require 'domain-probe/policy'
|
4
5
|
require 'domain-probe/guessing_policy'
|
5
6
|
require 'domain-probe/crawling_policy'
|
7
|
+
require 'domain-probe/thread_executor'
|
6
8
|
require 'set'
|
7
9
|
|
8
10
|
module Probe
|
9
11
|
|
10
12
|
class DomainProbe
|
11
|
-
|
12
|
-
MAX_RECURSIVE_QUERY_DEPTH = 6
|
13
13
|
|
14
14
|
@policies = []
|
15
15
|
|
@@ -32,213 +32,128 @@ module Probe
|
|
32
32
|
# collect the records under domain domain
|
33
33
|
#
|
34
34
|
def self.collect_records domain
|
35
|
-
self.new(
|
35
|
+
self.new().collect_records domain
|
36
36
|
end
|
37
37
|
|
38
38
|
#
|
39
39
|
# collect the ns records
|
40
40
|
#
|
41
41
|
def self.collect_ns_records domain
|
42
|
-
self.new(
|
42
|
+
self.new().collect_ns_records domain
|
43
43
|
end
|
44
|
-
|
45
|
-
#
|
46
|
-
# ----------------------------------------
|
47
|
-
#
|
48
|
-
#
|
49
|
-
#
|
50
|
-
def initialize(domain)
|
51
|
-
@domain = domain.downcase
|
52
|
-
@a_records = []
|
53
|
-
@aaaa_records = []
|
54
|
-
@cname_records = []
|
55
|
-
@mx_records = []
|
56
|
-
@mr_records = []
|
57
|
-
@txt_records = []
|
58
|
-
@hinfo_records = []
|
59
|
-
@srv_records = []
|
60
|
-
@spf_records = []
|
61
|
-
@ns_records = []
|
62
|
-
@semaphore = Mutex.new
|
63
|
-
end
|
64
|
-
|
65
|
-
attr_accessor :a_records
|
66
|
-
attr_accessor :aaaa_records
|
67
|
-
attr_accessor :cname_records
|
68
|
-
attr_accessor :mx_records
|
69
|
-
attr_accessor :mr_records
|
70
|
-
attr_accessor :txt_records
|
71
|
-
attr_accessor :hinfo_records
|
72
|
-
attr_accessor :srv_records
|
73
|
-
attr_accessor :ns_records
|
74
|
-
attr_accessor :spf_records
|
75
|
-
attr_accessor :semaphore
|
76
|
-
attr_accessor :domain
|
77
44
|
|
78
45
|
#
|
79
46
|
# collect the records under domain domain
|
47
|
+
# IF zone detection fails, return nil?
|
80
48
|
#
|
81
|
-
def collect_records
|
82
|
-
raise ArgumentError, "Domain name is expected" unless
|
83
|
-
domain = zone = Util.detect_zone(
|
84
|
-
return
|
85
|
-
|
86
|
-
|
87
|
-
nameservers = Util.detect_nameservers zone
|
49
|
+
def collect_records domain
|
50
|
+
raise ArgumentError, "Domain name is expected" unless domain
|
51
|
+
domain = zone = Util.detect_zone(domain.downcase)
|
52
|
+
return {} if !zone || is_tld(zone)
|
53
|
+
|
54
|
+
nameservers = detect_nameservers zone
|
88
55
|
possible_domains = Set.new
|
89
56
|
|
90
57
|
DomainProbe.policies.each{ |policy|
|
91
58
|
possible_domains |= policy.possible_host_under_domain(zone)
|
92
59
|
}
|
60
|
+
|
61
|
+
executor = ThreadExecutor.new
|
93
62
|
|
94
|
-
#
|
63
|
+
#check to see if a wildcard record there? if yes, stop the probing
|
64
|
+
executor.submit_task(wildcard_domain_name(zone),Net::DNS::A,zone,nameservers)
|
65
|
+
executor.submit_task(wildcard_domain_name(zone),Net::DNS::AAAA,zone,nameservers)
|
66
|
+
executor.execute_util_finished
|
67
|
+
|
68
|
+
#start again
|
69
|
+
executor.start_threads
|
70
|
+
|
71
|
+
#A/AAAA/CNAME records
|
95
72
|
possible_domains.each{ |possible_host|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
#AAAA
|
100
|
-
threads << threaded_resolve(possible_domain,Net::DNS::AAAA,zone,nameservers)
|
101
|
-
}
|
73
|
+
executor.submit_task(possible_domain_name(possible_host,zone),Net::DNS::A,zone,nameservers)
|
74
|
+
executor.submit_task(possible_domain_name(possible_host,zone),Net::DNS::AAAA,zone,nameservers)
|
75
|
+
} unless !executor.a_records.empty? || !executor.aaaa_records.empty? || !executor.cname_records.empty?
|
102
76
|
|
103
77
|
#A/CNAME records
|
104
|
-
|
78
|
+
executor.submit_task(domain,Net::DNS::A,zone,nameservers)
|
105
79
|
#AAAA
|
106
|
-
|
80
|
+
executor.submit_task(domain,Net::DNS::AAAA,zone,nameservers)
|
107
81
|
|
108
82
|
#MX records
|
109
|
-
|
83
|
+
executor.submit_task(domain,Net::DNS::MX,zone,nameservers)
|
110
84
|
|
111
85
|
#MR records
|
112
|
-
|
86
|
+
executor.submit_task(domain,Net::DNS::MR,zone,nameservers)
|
113
87
|
|
114
88
|
#TXT records
|
115
|
-
|
89
|
+
executor.submit_task(domain,Net::DNS::TXT,zone,nameservers)
|
116
90
|
|
117
91
|
#HINFO records
|
118
|
-
|
92
|
+
executor.submit_task(domain,Net::DNS::HINFO,zone,nameservers)
|
119
93
|
|
120
94
|
#SRV records
|
121
|
-
|
95
|
+
executor.submit_task(domain,Net::DNS::SRV,zone,nameservers)
|
122
96
|
|
123
97
|
#SPF records
|
124
|
-
|
98
|
+
executor.submit_task(domain,Net::DNS::SPF,zone,nameservers)
|
99
|
+
|
100
|
+
#SOA records
|
101
|
+
executor.submit_task(domain,Net::DNS::SOA,zone,nameservers)
|
125
102
|
|
126
103
|
#NS records
|
127
|
-
|
128
|
-
|
129
|
-
threads.each do |t|
|
130
|
-
t.join
|
131
|
-
end
|
104
|
+
executor.submit_task(domain,Net::DNS::NS,zone,nameservers)
|
132
105
|
|
106
|
+
executor.execute_util_finished
|
107
|
+
|
133
108
|
results = {
|
134
|
-
"a" =>
|
135
|
-
"aaaa" =>
|
136
|
-
"cname" =>
|
137
|
-
"mx" =>
|
138
|
-
"mr" =>
|
139
|
-
"txt" =>
|
140
|
-
"hinfo" =>
|
141
|
-
"srv" =>
|
142
|
-
"ns" =>
|
143
|
-
"spf" =>
|
109
|
+
"a" => executor.a_records,
|
110
|
+
"aaaa" => executor.aaaa_records,
|
111
|
+
"cname" => executor.cname_records,
|
112
|
+
"mx" => executor.mx_records,
|
113
|
+
"mr" => executor.mr_records,
|
114
|
+
"txt" => executor.txt_records,
|
115
|
+
"hinfo" => executor.hinfo_records,
|
116
|
+
"srv" => executor.srv_records,
|
117
|
+
"ns" => executor.ns_records.select{|record| record[:name] != "@"},
|
118
|
+
"spf" => executor.spf_records,
|
119
|
+
"soa" => executor.soa_records,
|
120
|
+
"zone" => Util.remove_trailing_dot_if_any(zone)
|
144
121
|
}
|
145
122
|
end
|
146
123
|
|
147
|
-
def collect_ns_records
|
148
|
-
raise ArgumentError, "Domain name is expected" unless
|
149
|
-
domain = zone = Util.detect_zone(
|
150
|
-
return [] if !zone
|
124
|
+
def collect_ns_records domain
|
125
|
+
raise ArgumentError, "Domain name is expected" unless domain
|
126
|
+
domain = zone = Util.detect_zone(domain.downcase)
|
127
|
+
return [] if !zone || is_tld(zone)
|
151
128
|
|
152
|
-
|
153
|
-
nameservers = Util.detect_nameservers zone
|
129
|
+
executor = ThreadExecutor.new
|
130
|
+
nameservers = Util.detect_nameservers zone.split(".").last
|
154
131
|
|
155
132
|
#NS records
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
t.join
|
160
|
-
end
|
161
|
-
|
162
|
-
self.ns_records
|
133
|
+
executor.submit_task(domain,Net::DNS::NS,zone,nameservers)
|
134
|
+
executor.execute_util_finished
|
135
|
+
executor.ns_records
|
163
136
|
end
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
def resolve_recursive(domain,type,zone,nameservers,depth)
|
174
|
-
|
175
|
-
depth += 1
|
176
|
-
return if depth > MAX_RECURSIVE_QUERY_DEPTH
|
177
|
-
single_record_handler = lambda{ |record|
|
178
|
-
|
179
|
-
if record.name && (record.name == Util.append_trailing_dot_if_necessary(zone) || Util.is_subdomain?(record.name,zone))
|
180
|
-
#A little overhead here
|
181
|
-
self.semaphore.synchronize {
|
182
|
-
case record.type
|
183
|
-
when "A"
|
184
|
-
self.a_records << record.to_hash(zone) unless self.a_records.include? record.to_hash(zone)
|
185
|
-
when "AAAA"
|
186
|
-
self.aaaa_records << record.to_hash(zone) unless self.aaaa_records.include? record.to_hash(zone)
|
187
|
-
when "CNAME"
|
188
|
-
self.cname_records << record.to_hash(zone) unless self.cname_records.include? record.to_hash(zone)
|
189
|
-
when "NS"
|
190
|
-
self.ns_records << record.to_hash(zone) unless self.ns_records.include? record.to_hash(zone)
|
191
|
-
when "MX"
|
192
|
-
self.mx_records << record.to_hash(zone) unless self.mx_records.include? record.to_hash(zone)
|
193
|
-
when "MR"
|
194
|
-
self.mr_records << record.to_hash(zone) unless self.mr_records.include? record.to_hash(zone)
|
195
|
-
when "TXT"
|
196
|
-
self.txt_records << record.to_hash(zone) unless self.txt_records.include? record.to_hash(zone)
|
197
|
-
when "SRV"
|
198
|
-
self.srv_records << record.to_hash(zone) unless self.srv_records.include? record.to_hash(zone)
|
199
|
-
when "HINFO"
|
200
|
-
self.hinfo_records << record.to_hash(zone) unless self.hinfo_records.include? record.to_hash(zone)
|
201
|
-
when "SPF"
|
202
|
-
self.spf_records << record.to_hash(zone) unless self.spf_records.include? record.to_hash(zone)
|
203
|
-
else
|
204
|
-
#nothing to do?
|
205
|
-
end
|
206
|
-
}
|
207
|
-
end
|
208
|
-
}
|
209
|
-
|
210
|
-
#
|
211
|
-
begin
|
212
|
-
|
213
|
-
resolver = nameservers.empty? ? Net::DNS::Resolver.new : Net::DNS::Resolver.new(:nameserver => nameservers)
|
214
|
-
resolver.log_level = Net::DNS::ERROR
|
215
|
-
packet = resolver.query(domain, type)
|
216
|
-
return unless packet
|
217
|
-
|
218
|
-
if packet.header.auth? || !packet.answer.empty?
|
219
|
-
packet.answer.each &single_record_handler if packet.answer
|
220
|
-
packet.authority.each &single_record_handler if packet.authority
|
221
|
-
packet.additional.each &single_record_handler if packet.additional
|
222
|
-
elsif !packet.authority.empty?
|
223
|
-
#following down
|
224
|
-
nameservers = Util.organize_authorities(packet.authority,packet.additional)
|
225
|
-
#tail recursive
|
226
|
-
resolve_recursive(domain,type,zone,nameservers,depth)
|
227
|
-
end
|
228
|
-
|
229
|
-
rescue =>ex
|
230
|
-
puts "#{ex.class}: #{ex.message}"
|
231
|
-
end
|
137
|
+
|
138
|
+
private
|
139
|
+
|
140
|
+
def detect_nameservers domain
|
141
|
+
nameserver_names = []
|
142
|
+
collect_ns_records(domain).each{ |authority|
|
143
|
+
nameserver_names << authority[:nsdname]
|
144
|
+
}
|
145
|
+
Util.resolve_addresses(nameserver_names)
|
232
146
|
end
|
233
|
-
|
147
|
+
|
148
|
+
def possible_domain_name(host,zone)
|
149
|
+
host + "." + zone
|
150
|
+
end
|
151
|
+
|
234
152
|
#
|
235
|
-
#
|
153
|
+
# used to detect wildcard records
|
236
154
|
#
|
237
|
-
def
|
238
|
-
|
239
|
-
depth = 0
|
240
|
-
resolve_recursive(domain,type,zone,nameservers,depth)
|
241
|
-
}
|
155
|
+
def wildcard_domain_name(zone)
|
156
|
+
"*." + zone
|
242
157
|
end
|
243
158
|
|
244
159
|
register_policy GuessingPolicy.new
|
@@ -1,4 +1,3 @@
|
|
1
|
-
require 'domain-probe/policy'
|
2
1
|
module Probe
|
3
2
|
|
4
3
|
#
|
@@ -36,6 +35,7 @@ module Probe
|
|
36
35
|
dir
|
37
36
|
direct
|
38
37
|
discovery
|
38
|
+
ditu
|
39
39
|
docs
|
40
40
|
download
|
41
41
|
downloads
|
@@ -63,6 +63,7 @@ module Probe
|
|
63
63
|
love
|
64
64
|
m
|
65
65
|
mail
|
66
|
+
map
|
66
67
|
maps
|
67
68
|
media
|
68
69
|
message
|
@@ -87,6 +88,7 @@ module Probe
|
|
87
88
|
resource
|
88
89
|
resources
|
89
90
|
sg
|
91
|
+
shop
|
90
92
|
shopping
|
91
93
|
sms
|
92
94
|
smtp
|
@@ -7,6 +7,34 @@ require 'net/dns/resolver'
|
|
7
7
|
#
|
8
8
|
module Net
|
9
9
|
module DNS
|
10
|
+
|
11
|
+
#to support *
|
12
|
+
class Resolver
|
13
|
+
|
14
|
+
def valid?(name)
|
15
|
+
if name =~ /[^-\w\.\*]/
|
16
|
+
raise ArgumentError, "Invalid domain name #{name}"
|
17
|
+
else
|
18
|
+
true
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
#to support *
|
24
|
+
class Question
|
25
|
+
|
26
|
+
def check_name(name)
|
27
|
+
name.strip!
|
28
|
+
if name =~ /[^\w\.\-_\*]/
|
29
|
+
raise NameError, "Question name #{name.inspect} not valid"
|
30
|
+
else
|
31
|
+
name
|
32
|
+
end
|
33
|
+
rescue
|
34
|
+
raise NameError, "Question name #{name.inspect} not valid"
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
10
38
|
class RR
|
11
39
|
|
12
40
|
#
|
@@ -31,8 +59,19 @@ module Net
|
|
31
59
|
|
32
60
|
{:name => cooked_value, :ttl => ttl}
|
33
61
|
end
|
34
|
-
|
35
|
-
|
62
|
+
|
63
|
+
class SOA
|
64
|
+
def to_hash zone
|
65
|
+
super(zone).merge({ :mname => self.mname.to_s,
|
66
|
+
:rname => self.rname.to_s,
|
67
|
+
:serial => self.serial.to_s,
|
68
|
+
:refresh => self.refresh.to_s,
|
69
|
+
:retry => self.retry.to_s,
|
70
|
+
:expire => self.expire.to_s,
|
71
|
+
:minimum => self.minimum.to_s,
|
72
|
+
})
|
73
|
+
end
|
74
|
+
end
|
36
75
|
|
37
76
|
class A
|
38
77
|
def to_hash zone
|
@@ -0,0 +1,168 @@
|
|
1
|
+
require 'thread'
|
2
|
+
|
3
|
+
module Probe
|
4
|
+
|
5
|
+
class ThreadExecutor
|
6
|
+
|
7
|
+
MAX_RECURSIVE_QUERY_DEPTH = 6
|
8
|
+
#
|
9
|
+
# ----------------------------------------
|
10
|
+
#
|
11
|
+
#
|
12
|
+
#
|
13
|
+
def initialize(size = 10)
|
14
|
+
@a_records = []
|
15
|
+
@aaaa_records = []
|
16
|
+
@cname_records = []
|
17
|
+
@mx_records = []
|
18
|
+
@mr_records = []
|
19
|
+
@txt_records = []
|
20
|
+
@hinfo_records = []
|
21
|
+
@srv_records = []
|
22
|
+
@spf_records = []
|
23
|
+
@soa_records = []
|
24
|
+
@ns_records = []
|
25
|
+
@mutex = Mutex.new
|
26
|
+
@jobs = Queue.new
|
27
|
+
@workers = start_threads(size)
|
28
|
+
end
|
29
|
+
|
30
|
+
attr_accessor :a_records
|
31
|
+
attr_accessor :aaaa_records
|
32
|
+
attr_accessor :cname_records
|
33
|
+
attr_accessor :mx_records
|
34
|
+
attr_accessor :mr_records
|
35
|
+
attr_accessor :txt_records
|
36
|
+
attr_accessor :hinfo_records
|
37
|
+
attr_accessor :srv_records
|
38
|
+
attr_accessor :ns_records
|
39
|
+
attr_accessor :spf_records
|
40
|
+
attr_accessor :soa_records
|
41
|
+
attr_accessor :mutex
|
42
|
+
attr_accessor :domain
|
43
|
+
attr_accessor :jobs
|
44
|
+
attr_accessor :workers
|
45
|
+
|
46
|
+
def start_threads(size = 10)
|
47
|
+
|
48
|
+
raise "Live threads are still there" if self.workers && !self.workers.select{|t| t.status}.empty?
|
49
|
+
|
50
|
+
self.workers = Array.new(size) do |i|
|
51
|
+
Thread.new do
|
52
|
+
catch(:done) do
|
53
|
+
loop do
|
54
|
+
job, args = self.jobs.pop
|
55
|
+
job.call(*args)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
#
|
63
|
+
#
|
64
|
+
def execute_util_finished
|
65
|
+
|
66
|
+
self.workers.length.times do
|
67
|
+
post_task { throw :done }
|
68
|
+
end
|
69
|
+
|
70
|
+
self.workers.map(&:join)
|
71
|
+
end
|
72
|
+
|
73
|
+
#
|
74
|
+
#
|
75
|
+
def post_task(*args, &block)
|
76
|
+
@jobs << [block, args] if block_given?
|
77
|
+
end
|
78
|
+
|
79
|
+
def post_result record,zone
|
80
|
+
|
81
|
+
return unless record && record.kind_of?(Net::DNS::RR)
|
82
|
+
#A little overhead here
|
83
|
+
self.mutex.synchronize {
|
84
|
+
record_hash = record.to_hash(zone)
|
85
|
+
case record.type
|
86
|
+
when "A"
|
87
|
+
self.a_records << record_hash unless self.a_records.include? record_hash
|
88
|
+
when "AAAA"
|
89
|
+
self.aaaa_records << record_hash unless self.aaaa_records.include? record_hash
|
90
|
+
when "CNAME"
|
91
|
+
self.cname_records << record_hash unless self.cname_records.include? record_hash
|
92
|
+
when "NS"
|
93
|
+
self.ns_records << record_hash unless self.ns_records.include? record_hash
|
94
|
+
when "MX"
|
95
|
+
self.mx_records << record_hash unless self.mx_records.include? record_hash
|
96
|
+
when "MR"
|
97
|
+
self.mr_records << record_hash unless self.mr_records.include? record_hash
|
98
|
+
when "TXT"
|
99
|
+
self.txt_records << record_hash unless self.txt_records.include? record_hash
|
100
|
+
when "SRV"
|
101
|
+
self.srv_records << record_hash unless self.srv_records.include? record_hash
|
102
|
+
when "HINFO"
|
103
|
+
self.hinfo_records << record_hash unless self.hinfo_records.include? record_hash
|
104
|
+
when "SPF"
|
105
|
+
self.spf_records << record_hash unless self.spf_records.include? record_hash
|
106
|
+
when "SOA"
|
107
|
+
self.soa_records << record_hash unless self.soa_records.include? record_hash
|
108
|
+
else
|
109
|
+
#nothing to do?
|
110
|
+
end
|
111
|
+
}
|
112
|
+
end
|
113
|
+
|
114
|
+
#
|
115
|
+
#
|
116
|
+
def submit_task(domain,type,zone,nameservers)
|
117
|
+
post_task(domain,type,zone,nameservers) { |domain,type,zone,nameservers|
|
118
|
+
depth = 0
|
119
|
+
resolve_recursive(domain,type,zone,nameservers,depth)
|
120
|
+
}
|
121
|
+
end
|
122
|
+
|
123
|
+
private
|
124
|
+
|
125
|
+
#
|
126
|
+
# the method will be called under the threaded execution context
|
127
|
+
# Attention:
|
128
|
+
# 1. tail recursive call happened when we get an un-authorized reply
|
129
|
+
# 2. depth limit to avoid infinite recursion
|
130
|
+
#
|
131
|
+
def resolve_recursive(domain,type,zone,nameservers,depth)
|
132
|
+
|
133
|
+
return if nameservers.empty?
|
134
|
+
depth += 1
|
135
|
+
return if depth > MAX_RECURSIVE_QUERY_DEPTH
|
136
|
+
single_record_handler = lambda{ |record|
|
137
|
+
|
138
|
+
if record.name && (record.name == Util.append_trailing_dot_if_necessary(zone) || Util.is_subdomain?(record.name,zone))
|
139
|
+
post_result record,zone
|
140
|
+
end
|
141
|
+
}
|
142
|
+
|
143
|
+
#
|
144
|
+
begin
|
145
|
+
|
146
|
+
resolver = Net::DNS::Resolver.new(:nameserver => nameservers,:udp_timeout => 2)
|
147
|
+
resolver.log_level = Net::DNS::ERROR
|
148
|
+
packet = resolver.query(domain, type)
|
149
|
+
return unless packet && !packet.header.error?
|
150
|
+
|
151
|
+
if packet.header.auth? || !packet.answer.empty?
|
152
|
+
packet.answer.each &single_record_handler if packet.answer
|
153
|
+
packet.authority.each &single_record_handler if packet.authority
|
154
|
+
packet.additional.each &single_record_handler if packet.additional
|
155
|
+
elsif !packet.authority.empty?
|
156
|
+
#following down
|
157
|
+
nameservers = Util.organize_authorities(packet.authority,packet.additional)
|
158
|
+
#tail recursive
|
159
|
+
resolve_recursive(domain,type,zone,nameservers,depth)
|
160
|
+
end
|
161
|
+
|
162
|
+
rescue =>ex
|
163
|
+
puts "#{ex.class} happened: #{ex.message} while resolving #{domain} of type #{type} at nameserves #{nameservers.join(',')}\n"
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
end
|
168
|
+
end
|
data/lib/domain-probe/util.rb
CHANGED
@@ -1,7 +1,3 @@
|
|
1
|
-
require 'domain-probe/resolver_patch'
|
2
|
-
#
|
3
|
-
#
|
4
|
-
#
|
5
1
|
module Probe
|
6
2
|
|
7
3
|
#
|
@@ -21,19 +17,20 @@ module Probe
|
|
21
17
|
zone = nil
|
22
18
|
begin
|
23
19
|
names.size.times { |index|
|
24
|
-
|
25
|
-
|
20
|
+
resolver = Net::DNS::Resolver.new(:udp_timeout => 2)
|
21
|
+
resolver.log_level = Net::DNS::ERROR
|
22
|
+
packet = resolver.query(domain, Net::DNS::SOA)
|
26
23
|
packet.answer.each { |record|
|
27
24
|
next unless record.name == domain && record.type == "SOA"
|
28
25
|
zone = domain
|
29
26
|
break
|
30
|
-
} if packet && packet.answer
|
27
|
+
} if packet && !packet.header.error? && packet.answer
|
31
28
|
|
32
29
|
return remove_trailing_dot_if_any(zone) unless !zone
|
33
30
|
domain = domain.slice(names[index].length + 1,domain.length - names[index].length - 1)
|
34
31
|
}
|
35
32
|
rescue => ex
|
36
|
-
puts "#{ex.class}: #{ex.message}"
|
33
|
+
puts "#{ex.class} happened: #{ex.message} while detecting zone of #{domain}\n"
|
37
34
|
end
|
38
35
|
|
39
36
|
zone
|
@@ -49,21 +46,22 @@ module Probe
|
|
49
46
|
nameserver_ips = []
|
50
47
|
nameserver_names = []
|
51
48
|
begin
|
52
|
-
|
53
|
-
|
49
|
+
resolver = Net::DNS::Resolver.new(:udp_timeout => 2)
|
50
|
+
resolver.log_level = Net::DNS::ERROR
|
51
|
+
packet = resolver.query(zone, Net::DNS::NS)
|
54
52
|
packet.answer.each { |record|
|
55
53
|
next unless record.name == append_trailing_dot_if_necessary(zone) && record.type == "NS"
|
56
54
|
nameserver_names << record.nsdname
|
57
55
|
packet.additional.select{ |x| x.name == record.nsdname && x.type == "A" }.each{ |rr|
|
58
56
|
nameserver_ips << rr.address.to_s
|
59
57
|
}
|
60
|
-
} if packet && packet.answer
|
58
|
+
} if packet && !packet.header.error? && packet.answer
|
61
59
|
|
62
60
|
if nameserver_ips.empty?
|
63
61
|
nameserver_ips += resolve_addresses(nameserver_names)
|
64
62
|
end
|
65
63
|
rescue => ex
|
66
|
-
puts "#{ex.class}: #{ex.message}"
|
64
|
+
puts "#{ex.class} happened: #{ex.message} while detecting nameservers of #{zone}\n"
|
67
65
|
end
|
68
66
|
|
69
67
|
nameserver_ips
|
@@ -128,8 +126,6 @@ module Probe
|
|
128
126
|
domain.end_with?(zone_with_trailing_dot) && domain[domain.index(zone_with_trailing_dot) - 1,1] == "."
|
129
127
|
end
|
130
128
|
|
131
|
-
private
|
132
|
-
|
133
129
|
#
|
134
130
|
# resolve the names to ip addresses
|
135
131
|
#
|
@@ -138,16 +134,18 @@ module Probe
|
|
138
134
|
return [] unless names && !names.empty?
|
139
135
|
addresses = []
|
140
136
|
begin
|
141
|
-
|
137
|
+
resolver = Net::DNS::Resolver.new(:udp_timeout => 2)
|
138
|
+
resolver.log_level = Net::DNS::ERROR
|
142
139
|
names.each{ |name|
|
143
|
-
packet =
|
140
|
+
packet = resolver.query(name, Net::DNS::A)
|
141
|
+
next unless packet && !packet.header.error?
|
144
142
|
packet.answer.each { |record|
|
145
143
|
next unless record.type == "A"
|
146
144
|
addresses << record.address.to_s
|
147
145
|
}
|
148
146
|
}
|
149
147
|
rescue => ex
|
150
|
-
puts "#{ex.class}: #{ex.message}"
|
148
|
+
puts "#{ex.class} happened: #{ex.message} while resolving address of one of #{names.join(',')}\n"
|
151
149
|
end
|
152
150
|
addresses
|
153
151
|
end
|
data/lib/domain-probe/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: domain-probe
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-05-07 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rspec
|
16
|
-
requirement: &
|
16
|
+
requirement: &70212344261720 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: '0'
|
22
22
|
type: :development
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70212344261720
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: rake
|
27
|
-
requirement: &
|
27
|
+
requirement: &70212344260960 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ! '>='
|
@@ -32,10 +32,10 @@ dependencies:
|
|
32
32
|
version: '0'
|
33
33
|
type: :development
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *70212344260960
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
37
|
name: net-dns
|
38
|
-
requirement: &
|
38
|
+
requirement: &70212344260340 !ruby/object:Gem::Requirement
|
39
39
|
none: false
|
40
40
|
requirements:
|
41
41
|
- - ! '>='
|
@@ -43,10 +43,10 @@ dependencies:
|
|
43
43
|
version: '0'
|
44
44
|
type: :runtime
|
45
45
|
prerelease: false
|
46
|
-
version_requirements: *
|
46
|
+
version_requirements: *70212344260340
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: nokogiri
|
49
|
-
requirement: &
|
49
|
+
requirement: &70212344259440 !ruby/object:Gem::Requirement
|
50
50
|
none: false
|
51
51
|
requirements:
|
52
52
|
- - ! '>='
|
@@ -54,7 +54,7 @@ dependencies:
|
|
54
54
|
version: '0'
|
55
55
|
type: :runtime
|
56
56
|
prerelease: false
|
57
|
-
version_requirements: *
|
57
|
+
version_requirements: *70212344259440
|
58
58
|
description: A simple library to probe the dns records under specific domain, as many
|
59
59
|
as possilbe
|
60
60
|
email:
|
@@ -73,6 +73,7 @@ files:
|
|
73
73
|
- lib/domain-probe/guessing_policy.rb
|
74
74
|
- lib/domain-probe/policy.rb
|
75
75
|
- lib/domain-probe/resolver_patch.rb
|
76
|
+
- lib/domain-probe/thread_executor.rb
|
76
77
|
- lib/domain-probe/util.rb
|
77
78
|
- lib/domain-probe/version.rb
|
78
79
|
homepage: http://www.yottaa.com
|
@@ -89,7 +90,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
89
90
|
version: '0'
|
90
91
|
segments:
|
91
92
|
- 0
|
92
|
-
hash:
|
93
|
+
hash: -3690899378102268182
|
93
94
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
94
95
|
none: false
|
95
96
|
requirements:
|
@@ -98,7 +99,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
98
99
|
version: '0'
|
99
100
|
segments:
|
100
101
|
- 0
|
101
|
-
hash:
|
102
|
+
hash: -3690899378102268182
|
102
103
|
requirements: []
|
103
104
|
rubyforge_project: domain-probe
|
104
105
|
rubygems_version: 1.8.10
|