domain-probe 1.2.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/domain-probe.rb +77 -162
- data/lib/domain-probe/crawling_policy.rb +0 -2
- data/lib/domain-probe/guessing_policy.rb +3 -1
- data/lib/domain-probe/resolver_patch.rb +41 -2
- data/lib/domain-probe/thread_executor.rb +168 -0
- data/lib/domain-probe/util.rb +15 -17
- data/lib/domain-probe/version.rb +1 -1
- metadata +13 -12
data/lib/domain-probe.rb
CHANGED
@@ -1,15 +1,15 @@
|
|
1
|
+
require 'domain-probe/tld'
|
1
2
|
require 'domain-probe/resolver_patch'
|
2
3
|
require 'domain-probe/util'
|
3
4
|
require 'domain-probe/policy'
|
4
5
|
require 'domain-probe/guessing_policy'
|
5
6
|
require 'domain-probe/crawling_policy'
|
7
|
+
require 'domain-probe/thread_executor'
|
6
8
|
require 'set'
|
7
9
|
|
8
10
|
module Probe
|
9
11
|
|
10
12
|
class DomainProbe
|
11
|
-
|
12
|
-
MAX_RECURSIVE_QUERY_DEPTH = 6
|
13
13
|
|
14
14
|
@policies = []
|
15
15
|
|
@@ -32,213 +32,128 @@ module Probe
|
|
32
32
|
# collect the records under domain domain
|
33
33
|
#
|
34
34
|
def self.collect_records domain
|
35
|
-
self.new(
|
35
|
+
self.new().collect_records domain
|
36
36
|
end
|
37
37
|
|
38
38
|
#
|
39
39
|
# collect the ns records
|
40
40
|
#
|
41
41
|
def self.collect_ns_records domain
|
42
|
-
self.new(
|
42
|
+
self.new().collect_ns_records domain
|
43
43
|
end
|
44
|
-
|
45
|
-
#
|
46
|
-
# ----------------------------------------
|
47
|
-
#
|
48
|
-
#
|
49
|
-
#
|
50
|
-
def initialize(domain)
|
51
|
-
@domain = domain.downcase
|
52
|
-
@a_records = []
|
53
|
-
@aaaa_records = []
|
54
|
-
@cname_records = []
|
55
|
-
@mx_records = []
|
56
|
-
@mr_records = []
|
57
|
-
@txt_records = []
|
58
|
-
@hinfo_records = []
|
59
|
-
@srv_records = []
|
60
|
-
@spf_records = []
|
61
|
-
@ns_records = []
|
62
|
-
@semaphore = Mutex.new
|
63
|
-
end
|
64
|
-
|
65
|
-
attr_accessor :a_records
|
66
|
-
attr_accessor :aaaa_records
|
67
|
-
attr_accessor :cname_records
|
68
|
-
attr_accessor :mx_records
|
69
|
-
attr_accessor :mr_records
|
70
|
-
attr_accessor :txt_records
|
71
|
-
attr_accessor :hinfo_records
|
72
|
-
attr_accessor :srv_records
|
73
|
-
attr_accessor :ns_records
|
74
|
-
attr_accessor :spf_records
|
75
|
-
attr_accessor :semaphore
|
76
|
-
attr_accessor :domain
|
77
44
|
|
78
45
|
#
|
79
46
|
# collect the records under domain domain
|
47
|
+
# IF zone detection fails, return nil?
|
80
48
|
#
|
81
|
-
def collect_records
|
82
|
-
raise ArgumentError, "Domain name is expected" unless
|
83
|
-
domain = zone = Util.detect_zone(
|
84
|
-
return
|
85
|
-
|
86
|
-
|
87
|
-
nameservers = Util.detect_nameservers zone
|
49
|
+
def collect_records domain
|
50
|
+
raise ArgumentError, "Domain name is expected" unless domain
|
51
|
+
domain = zone = Util.detect_zone(domain.downcase)
|
52
|
+
return {} if !zone || is_tld(zone)
|
53
|
+
|
54
|
+
nameservers = detect_nameservers zone
|
88
55
|
possible_domains = Set.new
|
89
56
|
|
90
57
|
DomainProbe.policies.each{ |policy|
|
91
58
|
possible_domains |= policy.possible_host_under_domain(zone)
|
92
59
|
}
|
60
|
+
|
61
|
+
executor = ThreadExecutor.new
|
93
62
|
|
94
|
-
#
|
63
|
+
#check to see if a wildcard record there? if yes, stop the probing
|
64
|
+
executor.submit_task(wildcard_domain_name(zone),Net::DNS::A,zone,nameservers)
|
65
|
+
executor.submit_task(wildcard_domain_name(zone),Net::DNS::AAAA,zone,nameservers)
|
66
|
+
executor.execute_util_finished
|
67
|
+
|
68
|
+
#start again
|
69
|
+
executor.start_threads
|
70
|
+
|
71
|
+
#A/AAAA/CNAME records
|
95
72
|
possible_domains.each{ |possible_host|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
#AAAA
|
100
|
-
threads << threaded_resolve(possible_domain,Net::DNS::AAAA,zone,nameservers)
|
101
|
-
}
|
73
|
+
executor.submit_task(possible_domain_name(possible_host,zone),Net::DNS::A,zone,nameservers)
|
74
|
+
executor.submit_task(possible_domain_name(possible_host,zone),Net::DNS::AAAA,zone,nameservers)
|
75
|
+
} unless !executor.a_records.empty? || !executor.aaaa_records.empty? || !executor.cname_records.empty?
|
102
76
|
|
103
77
|
#A/CNAME records
|
104
|
-
|
78
|
+
executor.submit_task(domain,Net::DNS::A,zone,nameservers)
|
105
79
|
#AAAA
|
106
|
-
|
80
|
+
executor.submit_task(domain,Net::DNS::AAAA,zone,nameservers)
|
107
81
|
|
108
82
|
#MX records
|
109
|
-
|
83
|
+
executor.submit_task(domain,Net::DNS::MX,zone,nameservers)
|
110
84
|
|
111
85
|
#MR records
|
112
|
-
|
86
|
+
executor.submit_task(domain,Net::DNS::MR,zone,nameservers)
|
113
87
|
|
114
88
|
#TXT records
|
115
|
-
|
89
|
+
executor.submit_task(domain,Net::DNS::TXT,zone,nameservers)
|
116
90
|
|
117
91
|
#HINFO records
|
118
|
-
|
92
|
+
executor.submit_task(domain,Net::DNS::HINFO,zone,nameservers)
|
119
93
|
|
120
94
|
#SRV records
|
121
|
-
|
95
|
+
executor.submit_task(domain,Net::DNS::SRV,zone,nameservers)
|
122
96
|
|
123
97
|
#SPF records
|
124
|
-
|
98
|
+
executor.submit_task(domain,Net::DNS::SPF,zone,nameservers)
|
99
|
+
|
100
|
+
#SOA records
|
101
|
+
executor.submit_task(domain,Net::DNS::SOA,zone,nameservers)
|
125
102
|
|
126
103
|
#NS records
|
127
|
-
|
128
|
-
|
129
|
-
threads.each do |t|
|
130
|
-
t.join
|
131
|
-
end
|
104
|
+
executor.submit_task(domain,Net::DNS::NS,zone,nameservers)
|
132
105
|
|
106
|
+
executor.execute_util_finished
|
107
|
+
|
133
108
|
results = {
|
134
|
-
"a" =>
|
135
|
-
"aaaa" =>
|
136
|
-
"cname" =>
|
137
|
-
"mx" =>
|
138
|
-
"mr" =>
|
139
|
-
"txt" =>
|
140
|
-
"hinfo" =>
|
141
|
-
"srv" =>
|
142
|
-
"ns" =>
|
143
|
-
"spf" =>
|
109
|
+
"a" => executor.a_records,
|
110
|
+
"aaaa" => executor.aaaa_records,
|
111
|
+
"cname" => executor.cname_records,
|
112
|
+
"mx" => executor.mx_records,
|
113
|
+
"mr" => executor.mr_records,
|
114
|
+
"txt" => executor.txt_records,
|
115
|
+
"hinfo" => executor.hinfo_records,
|
116
|
+
"srv" => executor.srv_records,
|
117
|
+
"ns" => executor.ns_records.select{|record| record[:name] != "@"},
|
118
|
+
"spf" => executor.spf_records,
|
119
|
+
"soa" => executor.soa_records,
|
120
|
+
"zone" => Util.remove_trailing_dot_if_any(zone)
|
144
121
|
}
|
145
122
|
end
|
146
123
|
|
147
|
-
def collect_ns_records
|
148
|
-
raise ArgumentError, "Domain name is expected" unless
|
149
|
-
domain = zone = Util.detect_zone(
|
150
|
-
return [] if !zone
|
124
|
+
def collect_ns_records domain
|
125
|
+
raise ArgumentError, "Domain name is expected" unless domain
|
126
|
+
domain = zone = Util.detect_zone(domain.downcase)
|
127
|
+
return [] if !zone || is_tld(zone)
|
151
128
|
|
152
|
-
|
153
|
-
nameservers = Util.detect_nameservers zone
|
129
|
+
executor = ThreadExecutor.new
|
130
|
+
nameservers = Util.detect_nameservers zone.split(".").last
|
154
131
|
|
155
132
|
#NS records
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
t.join
|
160
|
-
end
|
161
|
-
|
162
|
-
self.ns_records
|
133
|
+
executor.submit_task(domain,Net::DNS::NS,zone,nameservers)
|
134
|
+
executor.execute_util_finished
|
135
|
+
executor.ns_records
|
163
136
|
end
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
def resolve_recursive(domain,type,zone,nameservers,depth)
|
174
|
-
|
175
|
-
depth += 1
|
176
|
-
return if depth > MAX_RECURSIVE_QUERY_DEPTH
|
177
|
-
single_record_handler = lambda{ |record|
|
178
|
-
|
179
|
-
if record.name && (record.name == Util.append_trailing_dot_if_necessary(zone) || Util.is_subdomain?(record.name,zone))
|
180
|
-
#A little overhead here
|
181
|
-
self.semaphore.synchronize {
|
182
|
-
case record.type
|
183
|
-
when "A"
|
184
|
-
self.a_records << record.to_hash(zone) unless self.a_records.include? record.to_hash(zone)
|
185
|
-
when "AAAA"
|
186
|
-
self.aaaa_records << record.to_hash(zone) unless self.aaaa_records.include? record.to_hash(zone)
|
187
|
-
when "CNAME"
|
188
|
-
self.cname_records << record.to_hash(zone) unless self.cname_records.include? record.to_hash(zone)
|
189
|
-
when "NS"
|
190
|
-
self.ns_records << record.to_hash(zone) unless self.ns_records.include? record.to_hash(zone)
|
191
|
-
when "MX"
|
192
|
-
self.mx_records << record.to_hash(zone) unless self.mx_records.include? record.to_hash(zone)
|
193
|
-
when "MR"
|
194
|
-
self.mr_records << record.to_hash(zone) unless self.mr_records.include? record.to_hash(zone)
|
195
|
-
when "TXT"
|
196
|
-
self.txt_records << record.to_hash(zone) unless self.txt_records.include? record.to_hash(zone)
|
197
|
-
when "SRV"
|
198
|
-
self.srv_records << record.to_hash(zone) unless self.srv_records.include? record.to_hash(zone)
|
199
|
-
when "HINFO"
|
200
|
-
self.hinfo_records << record.to_hash(zone) unless self.hinfo_records.include? record.to_hash(zone)
|
201
|
-
when "SPF"
|
202
|
-
self.spf_records << record.to_hash(zone) unless self.spf_records.include? record.to_hash(zone)
|
203
|
-
else
|
204
|
-
#nothing to do?
|
205
|
-
end
|
206
|
-
}
|
207
|
-
end
|
208
|
-
}
|
209
|
-
|
210
|
-
#
|
211
|
-
begin
|
212
|
-
|
213
|
-
resolver = nameservers.empty? ? Net::DNS::Resolver.new : Net::DNS::Resolver.new(:nameserver => nameservers)
|
214
|
-
resolver.log_level = Net::DNS::ERROR
|
215
|
-
packet = resolver.query(domain, type)
|
216
|
-
return unless packet
|
217
|
-
|
218
|
-
if packet.header.auth? || !packet.answer.empty?
|
219
|
-
packet.answer.each &single_record_handler if packet.answer
|
220
|
-
packet.authority.each &single_record_handler if packet.authority
|
221
|
-
packet.additional.each &single_record_handler if packet.additional
|
222
|
-
elsif !packet.authority.empty?
|
223
|
-
#following down
|
224
|
-
nameservers = Util.organize_authorities(packet.authority,packet.additional)
|
225
|
-
#tail recursive
|
226
|
-
resolve_recursive(domain,type,zone,nameservers,depth)
|
227
|
-
end
|
228
|
-
|
229
|
-
rescue =>ex
|
230
|
-
puts "#{ex.class}: #{ex.message}"
|
231
|
-
end
|
137
|
+
|
138
|
+
private
|
139
|
+
|
140
|
+
def detect_nameservers domain
|
141
|
+
nameserver_names = []
|
142
|
+
collect_ns_records(domain).each{ |authority|
|
143
|
+
nameserver_names << authority[:nsdname]
|
144
|
+
}
|
145
|
+
Util.resolve_addresses(nameserver_names)
|
232
146
|
end
|
233
|
-
|
147
|
+
|
148
|
+
def possible_domain_name(host,zone)
|
149
|
+
host + "." + zone
|
150
|
+
end
|
151
|
+
|
234
152
|
#
|
235
|
-
#
|
153
|
+
# used to detect wildcard records
|
236
154
|
#
|
237
|
-
def
|
238
|
-
|
239
|
-
depth = 0
|
240
|
-
resolve_recursive(domain,type,zone,nameservers,depth)
|
241
|
-
}
|
155
|
+
def wildcard_domain_name(zone)
|
156
|
+
"*." + zone
|
242
157
|
end
|
243
158
|
|
244
159
|
register_policy GuessingPolicy.new
|
@@ -1,4 +1,3 @@
|
|
1
|
-
require 'domain-probe/policy'
|
2
1
|
module Probe
|
3
2
|
|
4
3
|
#
|
@@ -36,6 +35,7 @@ module Probe
|
|
36
35
|
dir
|
37
36
|
direct
|
38
37
|
discovery
|
38
|
+
ditu
|
39
39
|
docs
|
40
40
|
download
|
41
41
|
downloads
|
@@ -63,6 +63,7 @@ module Probe
|
|
63
63
|
love
|
64
64
|
m
|
65
65
|
mail
|
66
|
+
map
|
66
67
|
maps
|
67
68
|
media
|
68
69
|
message
|
@@ -87,6 +88,7 @@ module Probe
|
|
87
88
|
resource
|
88
89
|
resources
|
89
90
|
sg
|
91
|
+
shop
|
90
92
|
shopping
|
91
93
|
sms
|
92
94
|
smtp
|
@@ -7,6 +7,34 @@ require 'net/dns/resolver'
|
|
7
7
|
#
|
8
8
|
module Net
|
9
9
|
module DNS
|
10
|
+
|
11
|
+
#to support *
|
12
|
+
class Resolver
|
13
|
+
|
14
|
+
def valid?(name)
|
15
|
+
if name =~ /[^-\w\.\*]/
|
16
|
+
raise ArgumentError, "Invalid domain name #{name}"
|
17
|
+
else
|
18
|
+
true
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
#to support *
|
24
|
+
class Question
|
25
|
+
|
26
|
+
def check_name(name)
|
27
|
+
name.strip!
|
28
|
+
if name =~ /[^\w\.\-_\*]/
|
29
|
+
raise NameError, "Question name #{name.inspect} not valid"
|
30
|
+
else
|
31
|
+
name
|
32
|
+
end
|
33
|
+
rescue
|
34
|
+
raise NameError, "Question name #{name.inspect} not valid"
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
10
38
|
class RR
|
11
39
|
|
12
40
|
#
|
@@ -31,8 +59,19 @@ module Net
|
|
31
59
|
|
32
60
|
{:name => cooked_value, :ttl => ttl}
|
33
61
|
end
|
34
|
-
|
35
|
-
|
62
|
+
|
63
|
+
class SOA
|
64
|
+
def to_hash zone
|
65
|
+
super(zone).merge({ :mname => self.mname.to_s,
|
66
|
+
:rname => self.rname.to_s,
|
67
|
+
:serial => self.serial.to_s,
|
68
|
+
:refresh => self.refresh.to_s,
|
69
|
+
:retry => self.retry.to_s,
|
70
|
+
:expire => self.expire.to_s,
|
71
|
+
:minimum => self.minimum.to_s,
|
72
|
+
})
|
73
|
+
end
|
74
|
+
end
|
36
75
|
|
37
76
|
class A
|
38
77
|
def to_hash zone
|
@@ -0,0 +1,168 @@
|
|
1
|
+
require 'thread'
|
2
|
+
|
3
|
+
module Probe
|
4
|
+
|
5
|
+
class ThreadExecutor
|
6
|
+
|
7
|
+
MAX_RECURSIVE_QUERY_DEPTH = 6
|
8
|
+
#
|
9
|
+
# ----------------------------------------
|
10
|
+
#
|
11
|
+
#
|
12
|
+
#
|
13
|
+
def initialize(size = 10)
|
14
|
+
@a_records = []
|
15
|
+
@aaaa_records = []
|
16
|
+
@cname_records = []
|
17
|
+
@mx_records = []
|
18
|
+
@mr_records = []
|
19
|
+
@txt_records = []
|
20
|
+
@hinfo_records = []
|
21
|
+
@srv_records = []
|
22
|
+
@spf_records = []
|
23
|
+
@soa_records = []
|
24
|
+
@ns_records = []
|
25
|
+
@mutex = Mutex.new
|
26
|
+
@jobs = Queue.new
|
27
|
+
@workers = start_threads(size)
|
28
|
+
end
|
29
|
+
|
30
|
+
attr_accessor :a_records
|
31
|
+
attr_accessor :aaaa_records
|
32
|
+
attr_accessor :cname_records
|
33
|
+
attr_accessor :mx_records
|
34
|
+
attr_accessor :mr_records
|
35
|
+
attr_accessor :txt_records
|
36
|
+
attr_accessor :hinfo_records
|
37
|
+
attr_accessor :srv_records
|
38
|
+
attr_accessor :ns_records
|
39
|
+
attr_accessor :spf_records
|
40
|
+
attr_accessor :soa_records
|
41
|
+
attr_accessor :mutex
|
42
|
+
attr_accessor :domain
|
43
|
+
attr_accessor :jobs
|
44
|
+
attr_accessor :workers
|
45
|
+
|
46
|
+
def start_threads(size = 10)
|
47
|
+
|
48
|
+
raise "Live threads are still there" if self.workers && !self.workers.select{|t| t.status}.empty?
|
49
|
+
|
50
|
+
self.workers = Array.new(size) do |i|
|
51
|
+
Thread.new do
|
52
|
+
catch(:done) do
|
53
|
+
loop do
|
54
|
+
job, args = self.jobs.pop
|
55
|
+
job.call(*args)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
#
|
63
|
+
#
|
64
|
+
def execute_util_finished
|
65
|
+
|
66
|
+
self.workers.length.times do
|
67
|
+
post_task { throw :done }
|
68
|
+
end
|
69
|
+
|
70
|
+
self.workers.map(&:join)
|
71
|
+
end
|
72
|
+
|
73
|
+
#
|
74
|
+
#
|
75
|
+
def post_task(*args, &block)
|
76
|
+
@jobs << [block, args] if block_given?
|
77
|
+
end
|
78
|
+
|
79
|
+
def post_result record,zone
|
80
|
+
|
81
|
+
return unless record && record.kind_of?(Net::DNS::RR)
|
82
|
+
#A little overhead here
|
83
|
+
self.mutex.synchronize {
|
84
|
+
record_hash = record.to_hash(zone)
|
85
|
+
case record.type
|
86
|
+
when "A"
|
87
|
+
self.a_records << record_hash unless self.a_records.include? record_hash
|
88
|
+
when "AAAA"
|
89
|
+
self.aaaa_records << record_hash unless self.aaaa_records.include? record_hash
|
90
|
+
when "CNAME"
|
91
|
+
self.cname_records << record_hash unless self.cname_records.include? record_hash
|
92
|
+
when "NS"
|
93
|
+
self.ns_records << record_hash unless self.ns_records.include? record_hash
|
94
|
+
when "MX"
|
95
|
+
self.mx_records << record_hash unless self.mx_records.include? record_hash
|
96
|
+
when "MR"
|
97
|
+
self.mr_records << record_hash unless self.mr_records.include? record_hash
|
98
|
+
when "TXT"
|
99
|
+
self.txt_records << record_hash unless self.txt_records.include? record_hash
|
100
|
+
when "SRV"
|
101
|
+
self.srv_records << record_hash unless self.srv_records.include? record_hash
|
102
|
+
when "HINFO"
|
103
|
+
self.hinfo_records << record_hash unless self.hinfo_records.include? record_hash
|
104
|
+
when "SPF"
|
105
|
+
self.spf_records << record_hash unless self.spf_records.include? record_hash
|
106
|
+
when "SOA"
|
107
|
+
self.soa_records << record_hash unless self.soa_records.include? record_hash
|
108
|
+
else
|
109
|
+
#nothing to do?
|
110
|
+
end
|
111
|
+
}
|
112
|
+
end
|
113
|
+
|
114
|
+
#
|
115
|
+
#
|
116
|
+
def submit_task(domain,type,zone,nameservers)
|
117
|
+
post_task(domain,type,zone,nameservers) { |domain,type,zone,nameservers|
|
118
|
+
depth = 0
|
119
|
+
resolve_recursive(domain,type,zone,nameservers,depth)
|
120
|
+
}
|
121
|
+
end
|
122
|
+
|
123
|
+
private
|
124
|
+
|
125
|
+
#
|
126
|
+
# the method will be called under the threaded execution context
|
127
|
+
# Attention:
|
128
|
+
# 1. tail recursive call happened when we get an un-authorized reply
|
129
|
+
# 2. depth limit to avoid infinite recursion
|
130
|
+
#
|
131
|
+
def resolve_recursive(domain,type,zone,nameservers,depth)
|
132
|
+
|
133
|
+
return if nameservers.empty?
|
134
|
+
depth += 1
|
135
|
+
return if depth > MAX_RECURSIVE_QUERY_DEPTH
|
136
|
+
single_record_handler = lambda{ |record|
|
137
|
+
|
138
|
+
if record.name && (record.name == Util.append_trailing_dot_if_necessary(zone) || Util.is_subdomain?(record.name,zone))
|
139
|
+
post_result record,zone
|
140
|
+
end
|
141
|
+
}
|
142
|
+
|
143
|
+
#
|
144
|
+
begin
|
145
|
+
|
146
|
+
resolver = Net::DNS::Resolver.new(:nameserver => nameservers,:udp_timeout => 2)
|
147
|
+
resolver.log_level = Net::DNS::ERROR
|
148
|
+
packet = resolver.query(domain, type)
|
149
|
+
return unless packet && !packet.header.error?
|
150
|
+
|
151
|
+
if packet.header.auth? || !packet.answer.empty?
|
152
|
+
packet.answer.each &single_record_handler if packet.answer
|
153
|
+
packet.authority.each &single_record_handler if packet.authority
|
154
|
+
packet.additional.each &single_record_handler if packet.additional
|
155
|
+
elsif !packet.authority.empty?
|
156
|
+
#following down
|
157
|
+
nameservers = Util.organize_authorities(packet.authority,packet.additional)
|
158
|
+
#tail recursive
|
159
|
+
resolve_recursive(domain,type,zone,nameservers,depth)
|
160
|
+
end
|
161
|
+
|
162
|
+
rescue =>ex
|
163
|
+
puts "#{ex.class} happened: #{ex.message} while resolving #{domain} of type #{type} at nameserves #{nameservers.join(',')}\n"
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
end
|
168
|
+
end
|
data/lib/domain-probe/util.rb
CHANGED
@@ -1,7 +1,3 @@
|
|
1
|
-
require 'domain-probe/resolver_patch'
|
2
|
-
#
|
3
|
-
#
|
4
|
-
#
|
5
1
|
module Probe
|
6
2
|
|
7
3
|
#
|
@@ -21,19 +17,20 @@ module Probe
|
|
21
17
|
zone = nil
|
22
18
|
begin
|
23
19
|
names.size.times { |index|
|
24
|
-
|
25
|
-
|
20
|
+
resolver = Net::DNS::Resolver.new(:udp_timeout => 2)
|
21
|
+
resolver.log_level = Net::DNS::ERROR
|
22
|
+
packet = resolver.query(domain, Net::DNS::SOA)
|
26
23
|
packet.answer.each { |record|
|
27
24
|
next unless record.name == domain && record.type == "SOA"
|
28
25
|
zone = domain
|
29
26
|
break
|
30
|
-
} if packet && packet.answer
|
27
|
+
} if packet && !packet.header.error? && packet.answer
|
31
28
|
|
32
29
|
return remove_trailing_dot_if_any(zone) unless !zone
|
33
30
|
domain = domain.slice(names[index].length + 1,domain.length - names[index].length - 1)
|
34
31
|
}
|
35
32
|
rescue => ex
|
36
|
-
puts "#{ex.class}: #{ex.message}"
|
33
|
+
puts "#{ex.class} happened: #{ex.message} while detecting zone of #{domain}\n"
|
37
34
|
end
|
38
35
|
|
39
36
|
zone
|
@@ -49,21 +46,22 @@ module Probe
|
|
49
46
|
nameserver_ips = []
|
50
47
|
nameserver_names = []
|
51
48
|
begin
|
52
|
-
|
53
|
-
|
49
|
+
resolver = Net::DNS::Resolver.new(:udp_timeout => 2)
|
50
|
+
resolver.log_level = Net::DNS::ERROR
|
51
|
+
packet = resolver.query(zone, Net::DNS::NS)
|
54
52
|
packet.answer.each { |record|
|
55
53
|
next unless record.name == append_trailing_dot_if_necessary(zone) && record.type == "NS"
|
56
54
|
nameserver_names << record.nsdname
|
57
55
|
packet.additional.select{ |x| x.name == record.nsdname && x.type == "A" }.each{ |rr|
|
58
56
|
nameserver_ips << rr.address.to_s
|
59
57
|
}
|
60
|
-
} if packet && packet.answer
|
58
|
+
} if packet && !packet.header.error? && packet.answer
|
61
59
|
|
62
60
|
if nameserver_ips.empty?
|
63
61
|
nameserver_ips += resolve_addresses(nameserver_names)
|
64
62
|
end
|
65
63
|
rescue => ex
|
66
|
-
puts "#{ex.class}: #{ex.message}"
|
64
|
+
puts "#{ex.class} happened: #{ex.message} while detecting nameservers of #{zone}\n"
|
67
65
|
end
|
68
66
|
|
69
67
|
nameserver_ips
|
@@ -128,8 +126,6 @@ module Probe
|
|
128
126
|
domain.end_with?(zone_with_trailing_dot) && domain[domain.index(zone_with_trailing_dot) - 1,1] == "."
|
129
127
|
end
|
130
128
|
|
131
|
-
private
|
132
|
-
|
133
129
|
#
|
134
130
|
# resolve the names to ip addresses
|
135
131
|
#
|
@@ -138,16 +134,18 @@ module Probe
|
|
138
134
|
return [] unless names && !names.empty?
|
139
135
|
addresses = []
|
140
136
|
begin
|
141
|
-
|
137
|
+
resolver = Net::DNS::Resolver.new(:udp_timeout => 2)
|
138
|
+
resolver.log_level = Net::DNS::ERROR
|
142
139
|
names.each{ |name|
|
143
|
-
packet =
|
140
|
+
packet = resolver.query(name, Net::DNS::A)
|
141
|
+
next unless packet && !packet.header.error?
|
144
142
|
packet.answer.each { |record|
|
145
143
|
next unless record.type == "A"
|
146
144
|
addresses << record.address.to_s
|
147
145
|
}
|
148
146
|
}
|
149
147
|
rescue => ex
|
150
|
-
puts "#{ex.class}: #{ex.message}"
|
148
|
+
puts "#{ex.class} happened: #{ex.message} while resolving address of one of #{names.join(',')}\n"
|
151
149
|
end
|
152
150
|
addresses
|
153
151
|
end
|
data/lib/domain-probe/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: domain-probe
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-05-07 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rspec
|
16
|
-
requirement: &
|
16
|
+
requirement: &70212344261720 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: '0'
|
22
22
|
type: :development
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70212344261720
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: rake
|
27
|
-
requirement: &
|
27
|
+
requirement: &70212344260960 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ! '>='
|
@@ -32,10 +32,10 @@ dependencies:
|
|
32
32
|
version: '0'
|
33
33
|
type: :development
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *70212344260960
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
37
|
name: net-dns
|
38
|
-
requirement: &
|
38
|
+
requirement: &70212344260340 !ruby/object:Gem::Requirement
|
39
39
|
none: false
|
40
40
|
requirements:
|
41
41
|
- - ! '>='
|
@@ -43,10 +43,10 @@ dependencies:
|
|
43
43
|
version: '0'
|
44
44
|
type: :runtime
|
45
45
|
prerelease: false
|
46
|
-
version_requirements: *
|
46
|
+
version_requirements: *70212344260340
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: nokogiri
|
49
|
-
requirement: &
|
49
|
+
requirement: &70212344259440 !ruby/object:Gem::Requirement
|
50
50
|
none: false
|
51
51
|
requirements:
|
52
52
|
- - ! '>='
|
@@ -54,7 +54,7 @@ dependencies:
|
|
54
54
|
version: '0'
|
55
55
|
type: :runtime
|
56
56
|
prerelease: false
|
57
|
-
version_requirements: *
|
57
|
+
version_requirements: *70212344259440
|
58
58
|
description: A simple library to probe the dns records under specific domain, as many
|
59
59
|
as possilbe
|
60
60
|
email:
|
@@ -73,6 +73,7 @@ files:
|
|
73
73
|
- lib/domain-probe/guessing_policy.rb
|
74
74
|
- lib/domain-probe/policy.rb
|
75
75
|
- lib/domain-probe/resolver_patch.rb
|
76
|
+
- lib/domain-probe/thread_executor.rb
|
76
77
|
- lib/domain-probe/util.rb
|
77
78
|
- lib/domain-probe/version.rb
|
78
79
|
homepage: http://www.yottaa.com
|
@@ -89,7 +90,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
89
90
|
version: '0'
|
90
91
|
segments:
|
91
92
|
- 0
|
92
|
-
hash:
|
93
|
+
hash: -3690899378102268182
|
93
94
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
94
95
|
none: false
|
95
96
|
requirements:
|
@@ -98,7 +99,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
98
99
|
version: '0'
|
99
100
|
segments:
|
100
101
|
- 0
|
101
|
-
hash:
|
102
|
+
hash: -3690899378102268182
|
102
103
|
requirements: []
|
103
104
|
rubyforge_project: domain-probe
|
104
105
|
rubygems_version: 1.8.10
|