domain-probe 1.0.0 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +32 -1
- data/lib/domain-probe.rb +65 -28
- data/lib/domain-probe/crawling_policy.rb +8 -9
- data/lib/domain-probe/resolver_patch.rb +100 -1
- data/lib/domain-probe/util.rb +4 -4
- data/lib/domain-probe/version.rb +1 -1
- metadata +16 -10
data/README.md
CHANGED
@@ -1,3 +1,34 @@
|
|
1
1
|
= Domain::Probe
|
2
2
|
|
3
|
-
Domain::Probe is a simple library to probe the DNS records under specific domain, as many as possible
|
3
|
+
Domain::Probe is a simple library to probe the DNS records under specific domain, as many as possible
|
4
|
+
|
5
|
+
== Features
|
6
|
+
|
7
|
+
* Queries from authority name servers
|
8
|
+
* Recursive resolving
|
9
|
+
* Zone name detection
|
10
|
+
|
11
|
+
|
12
|
+
== Requirements
|
13
|
+
|
14
|
+
* Ruby >= 1.8.7
|
15
|
+
|
16
|
+
|
17
|
+
== Install
|
18
|
+
|
19
|
+
Just use RubyGems:
|
20
|
+
|
21
|
+
$ gem install domain-probe
|
22
|
+
|
23
|
+
If you want to install from source, you can use Rake:
|
24
|
+
|
25
|
+
$ rake install
|
26
|
+
|
27
|
+
|
28
|
+
== How to use
|
29
|
+
|
30
|
+
1. require 'domain-probe'
|
31
|
+
2. include Probe
|
32
|
+
3. collect_records "www.facebook.com"
|
33
|
+
|
34
|
+
|
data/lib/domain-probe.rb
CHANGED
@@ -32,7 +32,14 @@ module Probe
|
|
32
32
|
# collect the records under domain domain
|
33
33
|
#
|
34
34
|
def self.collect_records domain
|
35
|
-
self.new.collect_records
|
35
|
+
self.new(domain).collect_records
|
36
|
+
end
|
37
|
+
|
38
|
+
#
|
39
|
+
# collect the ns records
|
40
|
+
#
|
41
|
+
def self.collect_ns_records domain
|
42
|
+
self.new(domain).collect_ns_records
|
36
43
|
end
|
37
44
|
|
38
45
|
#
|
@@ -40,7 +47,8 @@ module Probe
|
|
40
47
|
#
|
41
48
|
#
|
42
49
|
#
|
43
|
-
def initialize()
|
50
|
+
def initialize(domain)
|
51
|
+
@domain = domain.downcase
|
44
52
|
@a_records = []
|
45
53
|
@aaaa_records = []
|
46
54
|
@cname_records = []
|
@@ -49,6 +57,7 @@ module Probe
|
|
49
57
|
@txt_records = []
|
50
58
|
@hinfo_records = []
|
51
59
|
@srv_records = []
|
60
|
+
@spf_records = []
|
52
61
|
@ns_records = []
|
53
62
|
@semaphore = Mutex.new
|
54
63
|
end
|
@@ -62,14 +71,17 @@ module Probe
|
|
62
71
|
attr_accessor :hinfo_records
|
63
72
|
attr_accessor :srv_records
|
64
73
|
attr_accessor :ns_records
|
74
|
+
attr_accessor :spf_records
|
65
75
|
attr_accessor :semaphore
|
76
|
+
attr_accessor :domain
|
66
77
|
|
67
78
|
#
|
68
79
|
# collect the records under domain domain
|
69
80
|
#
|
70
|
-
def collect_records
|
71
|
-
raise ArgumentError, "Domain name is expected" unless domain
|
72
|
-
domain = zone = Util.detect_zone(domain)
|
81
|
+
def collect_records
|
82
|
+
raise ArgumentError, "Domain name is expected" unless self.domain
|
83
|
+
domain = zone = Util.detect_zone(self.domain)
|
84
|
+
return [] if !zone
|
73
85
|
|
74
86
|
threads = []
|
75
87
|
nameservers = Util.detect_nameservers zone
|
@@ -108,6 +120,9 @@ module Probe
|
|
108
120
|
#SRV records
|
109
121
|
threads << threaded_resolve(domain,Net::DNS::SRV,zone,nameservers)
|
110
122
|
|
123
|
+
#SPF records
|
124
|
+
threads << threaded_resolve(domain,Net::DNS::SPF,zone,nameservers)
|
125
|
+
|
111
126
|
#NS records
|
112
127
|
threads << threaded_resolve(domain,Net::DNS::NS,zone,nameservers)
|
113
128
|
|
@@ -115,7 +130,36 @@ module Probe
|
|
115
130
|
t.join
|
116
131
|
end
|
117
132
|
|
118
|
-
|
133
|
+
results = {
|
134
|
+
"a" => self.a_records,
|
135
|
+
"aaaa" => self.aaaa_records,
|
136
|
+
"cname" => self.cname_records,
|
137
|
+
"mx" => self.mx_records,
|
138
|
+
"mr" => self.mr_records,
|
139
|
+
"txt" => self.txt_records,
|
140
|
+
"hinfo" => self.hinfo_records,
|
141
|
+
"srv" => self.srv_records,
|
142
|
+
"ns" => self.ns_records,
|
143
|
+
"spf" => self.spf_records
|
144
|
+
}
|
145
|
+
end
|
146
|
+
|
147
|
+
def collect_ns_records
|
148
|
+
raise ArgumentError, "Domain name is expected" unless self.domain
|
149
|
+
domain = zone = Util.detect_zone(self.domain)
|
150
|
+
return [] if !zone
|
151
|
+
|
152
|
+
threads = []
|
153
|
+
nameservers = Util.detect_nameservers zone
|
154
|
+
|
155
|
+
#NS records
|
156
|
+
threads << threaded_resolve(domain,Net::DNS::NS,zone,nameservers)
|
157
|
+
|
158
|
+
threads.each do |t|
|
159
|
+
t.join
|
160
|
+
end
|
161
|
+
|
162
|
+
self.ns_records
|
119
163
|
end
|
120
164
|
|
121
165
|
private
|
@@ -137,23 +181,25 @@ module Probe
|
|
137
181
|
self.semaphore.synchronize {
|
138
182
|
case record.type
|
139
183
|
when "A"
|
140
|
-
self.a_records << record unless self.a_records.include? record
|
184
|
+
self.a_records << record.to_hash(zone) unless self.a_records.include? record.to_hash(zone)
|
141
185
|
when "AAAA"
|
142
|
-
self.aaaa_records << record unless self.aaaa_records.include? record
|
186
|
+
self.aaaa_records << record.to_hash(zone) unless self.aaaa_records.include? record.to_hash(zone)
|
143
187
|
when "CNAME"
|
144
|
-
self.cname_records << record unless self.cname_records.include? record
|
188
|
+
self.cname_records << record.to_hash(zone) unless self.cname_records.include? record.to_hash(zone)
|
145
189
|
when "NS"
|
146
|
-
self.ns_records << record unless self.ns_records.include? record
|
190
|
+
self.ns_records << record.to_hash(zone) unless self.ns_records.include? record.to_hash(zone)
|
147
191
|
when "MX"
|
148
|
-
self.mx_records << record unless self.mx_records.include? record
|
192
|
+
self.mx_records << record.to_hash(zone) unless self.mx_records.include? record.to_hash(zone)
|
149
193
|
when "MR"
|
150
|
-
self.mr_records << record unless self.mr_records.include? record
|
194
|
+
self.mr_records << record.to_hash(zone) unless self.mr_records.include? record.to_hash(zone)
|
151
195
|
when "TXT"
|
152
|
-
self.txt_records << record unless self.txt_records.include? record
|
196
|
+
self.txt_records << record.to_hash(zone) unless self.txt_records.include? record.to_hash(zone)
|
153
197
|
when "SRV"
|
154
|
-
self.srv_records << record unless self.srv_records.include? record
|
198
|
+
self.srv_records << record.to_hash(zone) unless self.srv_records.include? record.to_hash(zone)
|
155
199
|
when "HINFO"
|
156
|
-
self.hinfo_records << record unless self.hinfo_records.include? record
|
200
|
+
self.hinfo_records << record.to_hash(zone) unless self.hinfo_records.include? record.to_hash(zone)
|
201
|
+
when "SPF"
|
202
|
+
self.spf_records << record.to_hash(zone) unless self.spf_records.include? record.to_hash(zone)
|
157
203
|
else
|
158
204
|
#nothing to do?
|
159
205
|
end
|
@@ -169,11 +215,11 @@ module Probe
|
|
169
215
|
packet = resolver.query(domain, type)
|
170
216
|
return unless packet
|
171
217
|
|
172
|
-
if packet.header.auth?
|
218
|
+
if packet.header.auth? || !packet.answer.empty?
|
173
219
|
packet.answer.each &single_record_handler if packet.answer
|
174
220
|
packet.authority.each &single_record_handler if packet.authority
|
175
221
|
packet.additional.each &single_record_handler if packet.additional
|
176
|
-
elsif
|
222
|
+
elsif !packet.authority.empty?
|
177
223
|
#following down
|
178
224
|
nameservers = Util.organize_authorities(packet.authority,packet.additional)
|
179
225
|
#tail recursive
|
@@ -206,17 +252,8 @@ module Probe
|
|
206
252
|
DomainProbe.collect_records domain
|
207
253
|
end
|
208
254
|
|
209
|
-
def
|
210
|
-
|
211
|
-
records = DomainProbe.collect_records(domain)
|
212
|
-
finish = Time.now.to_i
|
213
|
-
print "-----------------------","\n"
|
214
|
-
records.each{ |record|
|
215
|
-
print record, "\n"
|
216
|
-
}
|
217
|
-
print "-----------------------","\n"
|
218
|
-
print "Time elapsed:\t",finish - start,"seconds. \n"
|
219
|
-
nil
|
255
|
+
def collect_ns_records domain
|
256
|
+
DomainProbe.collect_ns_records domain
|
220
257
|
end
|
221
258
|
|
222
259
|
end
|
@@ -64,9 +64,6 @@ module Probe
|
|
64
64
|
|
65
65
|
private
|
66
66
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
67
|
#
|
71
68
|
#iterate the elements to find a set of host names
|
72
69
|
#
|
@@ -77,17 +74,19 @@ module Probe
|
|
77
74
|
|
78
75
|
links.each { |link|
|
79
76
|
next unless link[attribute]
|
80
|
-
|
77
|
+
attribute_value = link[attribute].downcase
|
78
|
+
head = attribute_value.index("//")
|
81
79
|
next unless head
|
82
|
-
tail =
|
83
|
-
next unless tail
|
80
|
+
tail = attribute_value.index(domain)
|
81
|
+
next unless tail && (tail > head)
|
84
82
|
|
85
83
|
#slice the host in the url
|
86
84
|
host = link[attribute].slice(head+DOUBLE_SLASH_LENGTH,tail-head-DOUBLE_SLASH_LENGTH)
|
87
85
|
|
88
|
-
|
89
|
-
|
90
|
-
|
86
|
+
next unless host.end_with? "."
|
87
|
+
|
88
|
+
host.chop!#remove the trailing dot if any
|
89
|
+
set.add host unless host.empty?
|
91
90
|
}
|
92
91
|
|
93
92
|
set
|
@@ -1,12 +1,93 @@
|
|
1
1
|
require 'net/dns/resolver'
|
2
2
|
|
3
3
|
#
|
4
|
-
# Hack:
|
4
|
+
# Hack:
|
5
|
+
# 1.the RR in net-dns didn't implement the ==,eql?,hash.methods
|
6
|
+
# 2.Monkey-patch to support SPF records
|
5
7
|
#
|
6
8
|
module Net
|
7
9
|
module DNS
|
8
10
|
class RR
|
11
|
+
|
12
|
+
#
|
13
|
+
# removing trailing dot, if there is one
|
14
|
+
#
|
15
|
+
def remove_trailing_dot_if_any domain
|
16
|
+
return domain.chop if domain && domain.end_with?(".")
|
17
|
+
domain
|
18
|
+
end
|
19
|
+
|
20
|
+
def to_hash zone
|
21
|
+
cooked_value = name
|
22
|
+
if zone
|
23
|
+
zone << "." unless zone.end_with? "."
|
24
|
+
if name == zone
|
25
|
+
cooked_value = "@"
|
26
|
+
else
|
27
|
+
index = name.index(zone)
|
28
|
+
cooked_value = name[0,index - 1] if index && index > 0
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
{:name => cooked_value, :ttl => ttl}
|
33
|
+
end
|
34
|
+
|
35
|
+
|
36
|
+
|
37
|
+
class A
|
38
|
+
def to_hash zone
|
39
|
+
super(zone).merge({:address => address.to_s})
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
class CNAME
|
44
|
+
def to_hash zone
|
45
|
+
super(zone).merge({:cname => remove_trailing_dot_if_any(cname)})
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
class MX
|
50
|
+
def to_hash zone
|
51
|
+
super(zone).merge({:exchange => remove_trailing_dot_if_any(exchange),:preference => preference.to_s})
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
class MR
|
56
|
+
def to_hash zone
|
57
|
+
super(zone).merge({:newname => remove_trailing_dot_if_any(newname)})
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
class HINFO
|
62
|
+
def to_hash zone
|
63
|
+
super(zone).merge({:cpu => cpu,:os => os})
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
class SRV
|
68
|
+
def to_hash zone
|
69
|
+
super(zone)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
class NS
|
74
|
+
def to_hash zone
|
75
|
+
super(zone).merge({:nsdname => remove_trailing_dot_if_any(nsdname)})
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
class AAAA
|
80
|
+
def to_hash zone
|
81
|
+
super(zone).merge({:address => address.to_s})
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
9
85
|
class TXT
|
86
|
+
|
87
|
+
def to_hash zone
|
88
|
+
super(zone).merge({:value => txt})
|
89
|
+
end
|
90
|
+
|
10
91
|
def value
|
11
92
|
@txt.to_s
|
12
93
|
end
|
@@ -29,6 +110,24 @@ module Net
|
|
29
110
|
def hash
|
30
111
|
self.data.hash
|
31
112
|
end
|
113
|
+
|
114
|
+
class SPF < Net::DNS::RR::TXT
|
115
|
+
|
116
|
+
private
|
117
|
+
def set_type
|
118
|
+
@type = Net::DNS::RR::Types.new("SPF")
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
class Types
|
123
|
+
TYPES['SPF'] = 99
|
124
|
+
end
|
125
|
+
|
126
|
+
end
|
127
|
+
|
128
|
+
module QueryTypes
|
129
|
+
SPF = 99
|
32
130
|
end
|
131
|
+
|
33
132
|
end
|
34
133
|
end
|
data/lib/domain-probe/util.rb
CHANGED
@@ -22,9 +22,9 @@ module Probe
|
|
22
22
|
begin
|
23
23
|
names.size.times { |index|
|
24
24
|
resovler = Net::DNS::Resolver.new
|
25
|
-
packet = resovler.query(domain, Net::DNS::
|
25
|
+
packet = resovler.query(domain, Net::DNS::SOA)
|
26
26
|
packet.answer.each { |record|
|
27
|
-
next unless record.name == domain && record.type == "
|
27
|
+
next unless record.name == domain && record.type == "SOA"
|
28
28
|
zone = domain
|
29
29
|
break
|
30
30
|
} if packet && packet.answer
|
@@ -114,8 +114,8 @@ module Probe
|
|
114
114
|
# removing trailing dot, if there is one
|
115
115
|
#
|
116
116
|
def self.remove_trailing_dot_if_any domain
|
117
|
-
return domain
|
118
|
-
domain
|
117
|
+
return domain.chop if domain && domain.end_with?(".")
|
118
|
+
domain
|
119
119
|
end
|
120
120
|
|
121
121
|
#
|
data/lib/domain-probe/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: domain-probe
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-04-
|
12
|
+
date: 2012-04-23 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rspec
|
16
|
-
requirement: &
|
16
|
+
requirement: &70267579051820 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: '0'
|
22
22
|
type: :development
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70267579051820
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: rake
|
27
|
-
requirement: &
|
27
|
+
requirement: &70267579051020 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ! '>='
|
@@ -32,10 +32,10 @@ dependencies:
|
|
32
32
|
version: '0'
|
33
33
|
type: :development
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *70267579051020
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
37
|
name: net-dns
|
38
|
-
requirement: &
|
38
|
+
requirement: &70267579050440 !ruby/object:Gem::Requirement
|
39
39
|
none: false
|
40
40
|
requirements:
|
41
41
|
- - ! '>='
|
@@ -43,10 +43,10 @@ dependencies:
|
|
43
43
|
version: '0'
|
44
44
|
type: :runtime
|
45
45
|
prerelease: false
|
46
|
-
version_requirements: *
|
46
|
+
version_requirements: *70267579050440
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: nokogiri
|
49
|
-
requirement: &
|
49
|
+
requirement: &70267579049620 !ruby/object:Gem::Requirement
|
50
50
|
none: false
|
51
51
|
requirements:
|
52
52
|
- - ! '>='
|
@@ -54,7 +54,7 @@ dependencies:
|
|
54
54
|
version: '0'
|
55
55
|
type: :runtime
|
56
56
|
prerelease: false
|
57
|
-
version_requirements: *
|
57
|
+
version_requirements: *70267579049620
|
58
58
|
description: A simple library to probe the dns records under specific domain, as many
|
59
59
|
as possilbe
|
60
60
|
email:
|
@@ -87,12 +87,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
87
87
|
- - ! '>='
|
88
88
|
- !ruby/object:Gem::Version
|
89
89
|
version: '0'
|
90
|
+
segments:
|
91
|
+
- 0
|
92
|
+
hash: 3749752069138402762
|
90
93
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
91
94
|
none: false
|
92
95
|
requirements:
|
93
96
|
- - ! '>='
|
94
97
|
- !ruby/object:Gem::Version
|
95
98
|
version: '0'
|
99
|
+
segments:
|
100
|
+
- 0
|
101
|
+
hash: 3749752069138402762
|
96
102
|
requirements: []
|
97
103
|
rubyforge_project: domain-probe
|
98
104
|
rubygems_version: 1.8.10
|