wmap 2.4.6 → 2.4.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2 -2
- data/bin/RHPG +24 -2
- data/bin/distrust +2 -3
- data/bin/trust +1 -1
- data/bin/trusts +1 -1
- data/bin/updateAll +12 -5
- data/bin/wadd +23 -4
- data/bin/wadds +23 -3
- data/bin/wdel +23 -4
- data/bin/wmap +10 -6
- data/bin/wmaps +3 -4
- data/demos/bruter.rb +1 -1
- data/demos/dns_brutes.rb +1 -1
- data/demos/filter_domain.rb +1 -1
- data/demos/filter_known_services.rb +2 -2
- data/demos/filter_prime.rb +1 -1
- data/demos/filter_site.rb +2 -2
- data/demos/filter_siteip.rb +1 -1
- data/demos/filter_url.rb +1 -1
- data/demos/new_fnd.rb +2 -2
- data/demos/site_format.rb +1 -1
- data/demos/whois_domain.rb +2 -2
- data/lib/wmap/dns_bruter.rb +4 -2
- data/lib/wmap/domain_tracker/sub_domain.rb +6 -3
- data/lib/wmap/domain_tracker.rb +3 -2
- data/lib/wmap/host_tracker/primary_host.rb +17 -13
- data/lib/wmap/host_tracker.rb +36 -29
- data/lib/wmap/site_tracker/deactivated_site.rb +2 -2
- data/lib/wmap/site_tracker.rb +25 -12
- data/lib/wmap/url_checker.rb +5 -4
- data/lib/wmap/utils/utils.rb +33 -33
- data/lib/wmap/wp_tracker.rb +75 -7
- data/lib/wmap.rb +12 -11
- data/logs/wmap.log +30 -0
- data/test/domain_tracker_test.rb +5 -5
- data/version.txt +2 -2
- data/wmap.gemspec +1 -1
- metadata +2 -2
data/lib/wmap/host_tracker.rb
CHANGED
@@ -11,7 +11,7 @@ require "singleton" # Implement singleton pattern to avoid race condition under
|
|
11
11
|
|
12
12
|
# Class to handle the local host data repository file where lists of known hosts from discovery and past assessment efforts are stored
|
13
13
|
class Wmap::HostTracker
|
14
|
-
|
14
|
+
include Singleton
|
15
15
|
include Wmap::Utils
|
16
16
|
|
17
17
|
attr_accessor :hosts_file, :max_parallel, :verbose, :data_dir
|
@@ -33,8 +33,8 @@ class Wmap::HostTracker
|
|
33
33
|
|
34
34
|
# Setter to load the known hosts from the local hosts file into a class instance
|
35
35
|
def load_known_hosts_from_file (f_hosts=@file_hosts)
|
36
|
-
|
37
|
-
|
36
|
+
#begin
|
37
|
+
puts "Loading local hosts from file: #{f_hosts} ..." if @verbose
|
38
38
|
known_hosts=Hash.new
|
39
39
|
@alias = Hash.new
|
40
40
|
f=File.open(f_hosts, 'r')
|
@@ -58,16 +58,16 @@ class Wmap::HostTracker
|
|
58
58
|
end
|
59
59
|
f.close
|
60
60
|
return known_hosts
|
61
|
-
rescue => ee
|
62
|
-
|
63
|
-
|
64
|
-
end
|
61
|
+
#rescue => ee
|
62
|
+
# puts "Exception on method #{__method__}: #{ee}"
|
63
|
+
# return known_hosts
|
64
|
+
#end
|
65
65
|
end
|
66
66
|
|
67
67
|
# Save the current local hosts hash table into a (random) data repository file
|
68
68
|
def save_known_hosts_to_file!(f_hosts=@file_hosts)
|
69
|
-
|
70
|
-
|
69
|
+
#begin
|
70
|
+
puts "Saving the local host repository from memory to file: #{f_hosts} ..."
|
71
71
|
timestamp=Time.now
|
72
72
|
f=File.open(f_hosts, 'w')
|
73
73
|
f.write "# local hosts file created by the #{self.class} class #{__method__} method at: #{timestamp}"
|
@@ -76,11 +76,12 @@ class Wmap::HostTracker
|
|
76
76
|
f.write "\n#{key}\t#{@known_hosts[key]}"
|
77
77
|
end
|
78
78
|
end
|
79
|
+
f.write "\n"
|
79
80
|
f.close
|
80
81
|
puts "local host repository is successfully saved to: #{f_hosts}"
|
81
|
-
rescue => ee
|
82
|
-
|
83
|
-
end
|
82
|
+
#rescue => ee
|
83
|
+
# puts "Exception on method #{__method__}: #{ee}"
|
84
|
+
#end
|
84
85
|
end
|
85
86
|
alias_method :save!, :save_known_hosts_to_file!
|
86
87
|
|
@@ -112,7 +113,11 @@ class Wmap::HostTracker
|
|
112
113
|
if is_ip?(ip)
|
113
114
|
# filter host to known domains only
|
114
115
|
root=get_domain_root(host)
|
115
|
-
|
116
|
+
puts "Domain root: #{root}" if @verbose
|
117
|
+
domain_tracker=Wmap::DomainTracker.instance
|
118
|
+
domain_tracker.data_dir=@data_dir
|
119
|
+
if domain_tracker.instance.domain_known?(root)
|
120
|
+
domain_tracker=nil
|
116
121
|
record[host]=ip
|
117
122
|
record[ip]=host
|
118
123
|
puts "Host data repository entry loaded: #{host} <=> #{ip}"
|
@@ -120,7 +125,8 @@ class Wmap::HostTracker
|
|
120
125
|
# add additional logic to update the sub-domain table as well, 02/10/2014
|
121
126
|
sub=get_sub_domain(host)
|
122
127
|
if sub!=root
|
123
|
-
tracker=Wmap::DomainTracker::SubDomain.
|
128
|
+
tracker=Wmap::DomainTracker::SubDomain.instance
|
129
|
+
tracker.data_dir=@data_dir
|
124
130
|
unless tracker.domain_known?(sub)
|
125
131
|
tracker.add(sub)
|
126
132
|
tracker.save!
|
@@ -130,6 +136,7 @@ class Wmap::HostTracker
|
|
130
136
|
@known_hosts.merge!(record)
|
131
137
|
return record
|
132
138
|
else
|
139
|
+
domain_tracker=nil
|
133
140
|
puts "Error - host #{host} has an untrusted internet root domain: #{root}\nPlease update the trusted domain seeds file first if necessary."
|
134
141
|
end
|
135
142
|
else
|
@@ -145,8 +152,8 @@ class Wmap::HostTracker
|
|
145
152
|
|
146
153
|
# Setter to add host entry to the local hosts in batch (from an array)
|
147
154
|
def bulk_add(list, num=@max_parallel)
|
148
|
-
|
149
|
-
|
155
|
+
#begin
|
156
|
+
puts "Add entries to the local host repository: #{list}"
|
150
157
|
results=Hash.new
|
151
158
|
if list.size > 0
|
152
159
|
puts "Start parallel host update processing on:\n #{list}" if @verbose
|
@@ -168,9 +175,9 @@ class Wmap::HostTracker
|
|
168
175
|
puts "Error: empty list - no entry is loaded. Please check your input list and try again."
|
169
176
|
end
|
170
177
|
return results
|
171
|
-
rescue => ee
|
172
|
-
puts "Exception on method #{__method__}: #{ee}"
|
173
|
-
end
|
178
|
+
# rescue => ee
|
179
|
+
# puts "Exception on method #{__method__}: #{ee}"
|
180
|
+
# end
|
174
181
|
end
|
175
182
|
alias_method :adds, :bulk_add
|
176
183
|
|
@@ -242,8 +249,8 @@ class Wmap::HostTracker
|
|
242
249
|
|
243
250
|
# Setter to refresh the entry from the cache one at a time
|
244
251
|
def refresh(host)
|
245
|
-
|
246
|
-
|
252
|
+
#begin
|
253
|
+
puts "Refresh the local host repository for host: #{host} "
|
247
254
|
host=host.strip.downcase
|
248
255
|
if @known_hosts.key?(host)
|
249
256
|
old_ip=@known_hosts[host]
|
@@ -266,15 +273,15 @@ class Wmap::HostTracker
|
|
266
273
|
else
|
267
274
|
puts "Error entry non exist: #{host}"
|
268
275
|
end
|
269
|
-
rescue => ee
|
270
|
-
|
271
|
-
end
|
276
|
+
#rescue => ee
|
277
|
+
# puts "Exception on method #{__method__}: #{ee}"
|
278
|
+
#end
|
272
279
|
end
|
273
280
|
|
274
281
|
# Refresh all the entries in the local hosts by querying the Internet
|
275
282
|
def refresh_all
|
276
283
|
puts "Refresh all the entries in the local host repository in one shot."
|
277
|
-
begin
|
284
|
+
#begin
|
278
285
|
changes=Hash.new
|
279
286
|
hosts=@known_hosts.keys
|
280
287
|
@known_hosts=Hash.new
|
@@ -290,9 +297,9 @@ class Wmap::HostTracker
|
|
290
297
|
#changes.map { |x| puts x }
|
291
298
|
puts "Done refreshing the local hosts."
|
292
299
|
return changes
|
293
|
-
rescue => ee
|
294
|
-
|
295
|
-
end
|
300
|
+
#rescue => ee
|
301
|
+
# puts "Exception on method #{__method__}: #{ee}"
|
302
|
+
#end
|
296
303
|
end
|
297
304
|
|
298
305
|
# Extract known root domains from the local host repository @known_hosts
|
@@ -439,7 +446,7 @@ class Wmap::HostTracker
|
|
439
446
|
sub = get_subdomain(hostname)
|
440
447
|
subs.push(sub) unless sub.nil?
|
441
448
|
end
|
442
|
-
subs.uniq!.sort!
|
449
|
+
subs.uniq!.sort! unless subs.empty?
|
443
450
|
puts "Found sub domains: #{subs}" if @verbose
|
444
451
|
return subs
|
445
452
|
rescue Exception => ee
|
@@ -5,7 +5,7 @@
|
|
5
5
|
#
|
6
6
|
# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
|
7
7
|
#++
|
8
|
-
|
8
|
+
require "singleton"
|
9
9
|
|
10
10
|
|
11
11
|
# Class to trace de-activated site. This is need for basic state tracking for our sites.
|
@@ -15,7 +15,7 @@ class SiteTracker
|
|
15
15
|
|
16
16
|
class DeactivatedSite < Wmap::SiteTracker
|
17
17
|
include Wmap::Utils
|
18
|
-
|
18
|
+
include Singleton
|
19
19
|
|
20
20
|
attr_accessor :sites_file, :known_sites, :verbose, :data_dir
|
21
21
|
|
data/lib/wmap/site_tracker.rb
CHANGED
@@ -6,14 +6,14 @@
|
|
6
6
|
# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
|
7
7
|
#++
|
8
8
|
require "parallel"
|
9
|
-
|
9
|
+
require "singleton"
|
10
10
|
require "nokogiri"
|
11
11
|
|
12
12
|
|
13
13
|
# Main class to automatically track the site inventory
|
14
14
|
class Wmap::SiteTracker
|
15
15
|
include Wmap::Utils
|
16
|
-
|
16
|
+
include Singleton
|
17
17
|
|
18
18
|
attr_accessor :sites_file, :max_parallel, :verbose, :data_dir
|
19
19
|
attr_reader :known_sites
|
@@ -116,7 +116,8 @@ class Wmap::SiteTracker
|
|
116
116
|
host=url_2_host(site)
|
117
117
|
ip=host_2_ip(host)
|
118
118
|
# Additional logic to refresh deactivated site, 02/12/2014
|
119
|
-
deact=Wmap::SiteTracker::DeactivatedSite.
|
119
|
+
deact=Wmap::SiteTracker::DeactivatedSite.instance
|
120
|
+
deact.data_dir=@data_dir
|
120
121
|
# only trust either the domain or IP we know
|
121
122
|
if is_ip?(host)
|
122
123
|
trusted=Wmap::CidrTracker.new(:data_dir=>@data_dir).ip_trusted?(ip)
|
@@ -125,7 +126,10 @@ class Wmap::SiteTracker
|
|
125
126
|
if root.nil?
|
126
127
|
raise "Invalid web site format. Please check your record again."
|
127
128
|
else
|
128
|
-
|
129
|
+
domain_tracker=Wmap::DomainTracker.instance
|
130
|
+
domain_tracker.data_dir=@data_dir
|
131
|
+
trusted=domain_tracker.domain_known?(root)
|
132
|
+
domain_tracker=nil
|
129
133
|
end
|
130
134
|
end
|
131
135
|
# add record only if trusted
|
@@ -140,7 +144,8 @@ class Wmap::SiteTracker
|
|
140
144
|
raise "Site is currently down. Skip #{site}" if checker['code']==10000
|
141
145
|
end
|
142
146
|
raise "Exception on add method - Fail to resolve the host-name: Host - #{host}, IP - #{ip}. Skip #{site}" unless is_ip?(ip)
|
143
|
-
my_tracker = Wmap::HostTracker.
|
147
|
+
my_tracker = Wmap::HostTracker.instance
|
148
|
+
my_tracker.data_dir=@data_dir
|
144
149
|
# Update the local host table when necessary
|
145
150
|
if is_ip?(host)
|
146
151
|
# Case #1: Trusted site contains IP
|
@@ -156,12 +161,15 @@ class Wmap::SiteTracker
|
|
156
161
|
host1=ip_2_host(host)
|
157
162
|
puts "host1: #{host1}" if @verbose
|
158
163
|
if is_fqdn?(host1)
|
159
|
-
|
164
|
+
host_tracker=Wmap::HostTracker.instance
|
165
|
+
host_tracker.data_dir=@data_dir
|
166
|
+
if host_tracker.domain_known?(host1)
|
160
167
|
# replace IP with host-name only if domain root is known
|
161
168
|
puts "Host found from the Internet reverse DNS lookup for #{ip}: #{host1}" if @verbose
|
162
169
|
host=host1
|
163
170
|
site.sub!(/\d+\.\d+\.\d+\.\d+/,host)
|
164
171
|
end
|
172
|
+
host_tracker=nil
|
165
173
|
end
|
166
174
|
end
|
167
175
|
# Adding site for Case #1
|
@@ -288,7 +296,8 @@ class Wmap::SiteTracker
|
|
288
296
|
puts "Remove entry from the site store: #{site} " if @verbose
|
289
297
|
begin
|
290
298
|
# Additional logic to deactivate the site properly, by moving it to the DeactivatedSite list, 02/07/2014
|
291
|
-
deact=Wmap::SiteTracker::DeactivatedSite.
|
299
|
+
deact=Wmap::SiteTracker::DeactivatedSite.instance
|
300
|
+
deact.data_dir=@data_dir
|
292
301
|
site=site.strip.downcase
|
293
302
|
site=url_2_site(site)
|
294
303
|
if @known_sites.key?(site)
|
@@ -574,10 +583,11 @@ class Wmap::SiteTracker
|
|
574
583
|
def get_uniq_sites
|
575
584
|
puts "Getter to retrieve unique sites containing unique IP:PORT key identifier." if @verbose
|
576
585
|
begin
|
577
|
-
#primary_host_tracker=Wmap::HostTracker::PrimaryHost.
|
586
|
+
#primary_host_tracker=Wmap::HostTracker::PrimaryHost.instance
|
578
587
|
sites=Hash.new
|
579
588
|
#uniqueness=Hash.new
|
580
|
-
my_tracker=Wmap::HostTracker.
|
589
|
+
my_tracker=Wmap::HostTracker.instance
|
590
|
+
my_tracker.data_dir=@data_dir
|
581
591
|
@known_sites.keys.map do |key|
|
582
592
|
port=url_2_port(key).to_s
|
583
593
|
host=url_2_host(key)
|
@@ -677,7 +687,8 @@ class Wmap::SiteTracker
|
|
677
687
|
begin
|
678
688
|
updates=Array.new
|
679
689
|
sites=get_ip_sites
|
680
|
-
my_tracker=Wmap::HostTracker.
|
690
|
+
my_tracker=Wmap::HostTracker.instance
|
691
|
+
my_tracker.data_dir=@data_dir
|
681
692
|
sites.map do |site|
|
682
693
|
puts "Work on resolve the IP site: #{site}" if @verbose
|
683
694
|
ip=url_2_host(site)
|
@@ -837,8 +848,10 @@ class Wmap::SiteTracker
|
|
837
848
|
def get_prim_uniq_sites
|
838
849
|
puts "Retrieve and prime unique sites in the site store. " if @verbose
|
839
850
|
#begin
|
840
|
-
host_tracker=Wmap::HostTracker.
|
841
|
-
|
851
|
+
host_tracker=Wmap::HostTracker.instance
|
852
|
+
host_tracker.data_dir=@data_dir
|
853
|
+
primary_host_tracker=Wmap::HostTracker::PrimaryHost.instance
|
854
|
+
primary_host_tracker.data_dir=@data_dir
|
842
855
|
# Step 1. Retrieve the unique site list first
|
843
856
|
sites=get_uniq_sites
|
844
857
|
prim_uniq_sites=Array.new
|
data/lib/wmap/url_checker.rb
CHANGED
@@ -176,9 +176,9 @@ class Wmap::UrlChecker
|
|
176
176
|
|
177
177
|
# Test the URL / site and return the redirection location (3xx response code only)
|
178
178
|
def redirect_location (url)
|
179
|
-
puts "Test the redirection location for the url: #{url}" if @verbose
|
180
|
-
location=""
|
181
179
|
begin
|
180
|
+
puts "Test the redirection location for the url: #{url}" if @verbose
|
181
|
+
location=""
|
182
182
|
raise "Invalid url: #{url}" unless is_url?(url)
|
183
183
|
url=url.strip.downcase
|
184
184
|
timeo = @http_timeout/1000.0
|
@@ -196,6 +196,7 @@ class Wmap::UrlChecker
|
|
196
196
|
end
|
197
197
|
request = Net::HTTP::Get.new(uri.request_uri)
|
198
198
|
response = http.request(request)
|
199
|
+
puts "Response: #{response}" if @verbose
|
199
200
|
case response
|
200
201
|
when Net::HTTPRedirection then
|
201
202
|
location = response['location']
|
@@ -213,9 +214,9 @@ class Wmap::UrlChecker
|
|
213
214
|
|
214
215
|
# Test the URL / site and return the web server type from the HTTP header "server" field
|
215
216
|
def get_server_header (url)
|
216
|
-
puts "Retrieve the server header field from the url: #{url}" if @verbose
|
217
|
-
server=String.new
|
218
217
|
begin
|
218
|
+
puts "Retrieve the server header field from the url: #{url}" if @verbose
|
219
|
+
server=String.new
|
219
220
|
raise "Invalid url: #{url}" unless is_url?(url)
|
220
221
|
url=url.strip.downcase
|
221
222
|
timeo = @http_timeout/1000.0
|
data/lib/wmap/utils/utils.rb
CHANGED
@@ -11,12 +11,12 @@ require "netaddr"
|
|
11
11
|
|
12
12
|
# Main utility module to provide the common functions across different classes
|
13
13
|
module Wmap
|
14
|
-
module Utils
|
14
|
+
module Utils
|
15
15
|
include Wmap::Utils::DomainRoot
|
16
16
|
include Wmap::Utils::UrlMagic
|
17
17
|
include Wmap::Utils::Logger
|
18
18
|
extend self
|
19
|
-
|
19
|
+
|
20
20
|
# Load entries from a text file and return an array
|
21
21
|
def file_2_list(f,lc=true)
|
22
22
|
puts "Loading records from file: #{f}" if @verbose
|
@@ -27,7 +27,7 @@ module Wmap
|
|
27
27
|
line=line.chomp.strip
|
28
28
|
next if line.nil?
|
29
29
|
next if line.empty?
|
30
|
-
next if line =~ /^\s*#/
|
30
|
+
next if line =~ /^\s*#/
|
31
31
|
line=line.downcase if lc==true
|
32
32
|
list.push(line.chomp.strip)
|
33
33
|
end
|
@@ -55,7 +55,7 @@ module Wmap
|
|
55
55
|
return nil
|
56
56
|
end
|
57
57
|
end
|
58
|
-
|
58
|
+
|
59
59
|
# Load entries from a text file and return a hash
|
60
60
|
def file_2_hash(f,lc=true)
|
61
61
|
puts "Loading records from file: #{f}" if @verbose
|
@@ -67,7 +67,7 @@ module Wmap
|
|
67
67
|
next if line.nil?
|
68
68
|
next if line.empty?
|
69
69
|
line=line.downcase if lc==true
|
70
|
-
next if line =~ /^\s*#/
|
70
|
+
next if line =~ /^\s*#/
|
71
71
|
hs[line]=true unless hs.key?(line)
|
72
72
|
end
|
73
73
|
file.close
|
@@ -77,7 +77,7 @@ module Wmap
|
|
77
77
|
return nil
|
78
78
|
end
|
79
79
|
end
|
80
|
-
|
80
|
+
|
81
81
|
# Query the name-server to see if the dns record is still valid
|
82
82
|
def valid_dns_record? (hostname)
|
83
83
|
puts "Validate the hostname record: #{hostname}" if @verbose
|
@@ -99,7 +99,7 @@ module Wmap
|
|
99
99
|
|
100
100
|
# Test the DNS server if zone transfer is allowed. If allowed, save the found hosts into the class variable.
|
101
101
|
def zone_transferable?(domain)
|
102
|
-
puts "Check if the domain allows free zone transfer: #{domain}"
|
102
|
+
puts "Check if the domain allows free zone transfer: #{domain}"
|
103
103
|
begin
|
104
104
|
transferable=false
|
105
105
|
domain=domain.downcase
|
@@ -108,32 +108,32 @@ module Wmap
|
|
108
108
|
puts "Retrieved name servers: #{nameservers}" if @verbose
|
109
109
|
nameservers.each do |nssrv|
|
110
110
|
begin
|
111
|
-
puts "Attempt zone transfer on name server: #{nssrv}"
|
111
|
+
puts "Attempt zone transfer on name server: #{nssrv}"
|
112
112
|
if nssrv.nil?
|
113
113
|
abort "Method input variable error: no name server found!" if @verbose
|
114
114
|
next
|
115
|
-
end
|
115
|
+
end
|
116
116
|
zt = Dnsruby::ZoneTransfer.new
|
117
|
-
zt.server=(nssrv) if nssrv!=""
|
117
|
+
zt.server=(nssrv) if nssrv!=""
|
118
118
|
records = zt.transfer(domain)
|
119
119
|
if records==nil
|
120
|
-
puts "#{domain} zone transfer is not allowed on name server: #{nssrv}"
|
120
|
+
puts "#{domain} zone transfer is not allowed on name server: #{nssrv}"
|
121
121
|
next
|
122
122
|
else
|
123
123
|
puts "#{domain} zone transfer is allowed!"
|
124
124
|
transferable=true
|
125
125
|
end
|
126
126
|
rescue Exception=>ee
|
127
|
-
puts "Exception on method zone_transferable? for domain #{domain}: #{ee}"
|
127
|
+
puts "Exception on method zone_transferable? for domain #{domain}: #{ee}"
|
128
128
|
end
|
129
129
|
end
|
130
130
|
return transferable
|
131
131
|
rescue Exception => ee
|
132
132
|
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
133
|
-
return
|
134
|
-
end
|
135
|
-
end
|
136
|
-
|
133
|
+
return false
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
137
|
# Test if it's a legitimate IP4 address
|
138
138
|
def is_ip? (ip)
|
139
139
|
puts "Validate the IP format is valid: #{ip}" if @verbose
|
@@ -155,8 +155,8 @@ module Wmap
|
|
155
155
|
end
|
156
156
|
end
|
157
157
|
alias_method :is_valid_ip?, :is_ip?
|
158
|
-
|
159
|
-
# Simple test a host string format. Return true if it contains a valid internet domain sub-string. Note: Don't be confused with another method 'valid_dns_record?', which is a stricter and time-consuming test on the DNS server for a resolvable internet host.
|
158
|
+
|
159
|
+
# Simple test a host string format. Return true if it contains a valid internet domain sub-string. Note: Don't be confused with another method 'valid_dns_record?', which is a stricter and time-consuming test on the DNS server for a resolvable internet host.
|
160
160
|
def is_fqdn? (host)
|
161
161
|
puts "Validate the host-name format is valid: #{host}" if @verbose
|
162
162
|
begin
|
@@ -202,17 +202,17 @@ module Wmap
|
|
202
202
|
return false
|
203
203
|
end
|
204
204
|
end
|
205
|
-
|
205
|
+
|
206
206
|
# Sort an array of IPs in the ascendant order
|
207
207
|
def sort_ips (ips)
|
208
208
|
begin
|
209
209
|
"Sort the list of IP address in the ascendant order: #{ips}" if @verbose
|
210
210
|
return NetAddr.sort(ips)
|
211
|
-
rescue => ee
|
211
|
+
rescue => ee
|
212
212
|
puts "Exception on method sort_ips for IPs #{ips}: #{ee}" # if @verbose
|
213
|
-
end
|
213
|
+
end
|
214
214
|
end
|
215
|
-
|
215
|
+
|
216
216
|
# Perform the DNS query on a hostname over the Internet. Return the resolved IP(s) in an array
|
217
217
|
def host_2_ips (hostname)
|
218
218
|
begin
|
@@ -233,7 +233,7 @@ module Wmap
|
|
233
233
|
puts "Exception on method host_2_ips for host #{hostname}: #{ee}" if @verbose
|
234
234
|
return nil
|
235
235
|
end
|
236
|
-
end
|
236
|
+
end
|
237
237
|
alias_method :ns_lookup, :host_2_ips
|
238
238
|
|
239
239
|
# Perform DNS query on a hostname. Return the first resolved IP as a string
|
@@ -258,8 +258,8 @@ module Wmap
|
|
258
258
|
puts "Exception on method host_2_ip for host #{hostname}: #{ee}" if @verbose
|
259
259
|
return nil
|
260
260
|
end
|
261
|
-
end
|
262
|
-
|
261
|
+
end
|
262
|
+
|
263
263
|
# Retrieve a list of the authoritative name servers from the Internet whois data repository for the host / subdomain / domain
|
264
264
|
def get_nameservers (host)
|
265
265
|
puts "Retrieve a list of authoritative name server for: #{host}" if @verbose
|
@@ -269,7 +269,7 @@ module Wmap
|
|
269
269
|
ns = w.query(domain).nameservers.map! { |x| x.name }
|
270
270
|
if ns.empty?
|
271
271
|
puts "No name server found for domain root: #{domain}" if @verbose
|
272
|
-
return nil
|
272
|
+
return nil
|
273
273
|
else
|
274
274
|
return ns
|
275
275
|
end
|
@@ -288,7 +288,7 @@ module Wmap
|
|
288
288
|
ns = w.query(domain).nameservers.map! { |x| x.name }
|
289
289
|
if ns.empty?
|
290
290
|
puts "No name server found for domain root: #{domain}" if @verbose
|
291
|
-
return nil
|
291
|
+
return nil
|
292
292
|
else
|
293
293
|
return ns.first
|
294
294
|
end
|
@@ -298,21 +298,21 @@ module Wmap
|
|
298
298
|
end
|
299
299
|
end
|
300
300
|
alias_method :get_ns, :get_nameserver
|
301
|
-
|
301
|
+
|
302
302
|
# Perform reverse dns lookup for an IP. Return the found 'hostname' if found, or the original IP if not
|
303
303
|
def reverse_dns_lookup (ip)
|
304
304
|
puts "Retrieve the hostname by the reverse DNS lookup on IP: #{ip}"
|
305
305
|
hostname = ip
|
306
|
-
begin
|
306
|
+
begin
|
307
307
|
hostname = Socket.gethostbyaddr(ip.split('.').collect{ |x| x.to_i}.pack("CCCC"))[0]
|
308
308
|
return hostname.downcase
|
309
309
|
rescue => ee
|
310
310
|
puts "Exception on method reverse_dns_lookup: #{ee}" if @verbose
|
311
311
|
return hostname
|
312
|
-
end
|
312
|
+
end
|
313
313
|
end
|
314
314
|
alias_method :ip_2_host, :reverse_dns_lookup
|
315
|
-
|
315
|
+
|
316
316
|
# Convert a CIDR to a list of IPs: Input is a CIDR expression such as '192.168.1.1/30', output is an array of IPs
|
317
317
|
def cidr_2_ips (cidr)
|
318
318
|
puts "Method to convert a CIDR block into a list of IPs: #{cidr}" if @verbose
|
@@ -324,10 +324,10 @@ module Wmap
|
|
324
324
|
rescue => ee
|
325
325
|
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
326
326
|
return nil
|
327
|
-
end
|
327
|
+
end
|
328
328
|
end
|
329
329
|
|
330
330
|
|
331
|
-
|
331
|
+
|
332
332
|
end
|
333
333
|
end
|
data/lib/wmap/wp_tracker.rb
CHANGED
@@ -87,7 +87,7 @@ class Wmap::WpTracker
|
|
87
87
|
end
|
88
88
|
end
|
89
89
|
f.close
|
90
|
-
puts "
|
90
|
+
puts "WordPress site cache table is successfully saved: #{file_wps}"
|
91
91
|
rescue => ee
|
92
92
|
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
93
93
|
end
|
@@ -127,6 +127,10 @@ class Wmap::WpTracker
|
|
127
127
|
found=true
|
128
128
|
elsif wp_meta?(site)
|
129
129
|
found=true
|
130
|
+
elsif wp_login?(site)
|
131
|
+
found=true
|
132
|
+
elsif wp_rpc?(site)
|
133
|
+
found=true
|
130
134
|
else
|
131
135
|
found=false
|
132
136
|
end
|
@@ -141,7 +145,7 @@ class Wmap::WpTracker
|
|
141
145
|
#begin
|
142
146
|
puts "Add entries to the local cache table from site tracker: " if @verbose
|
143
147
|
results=Hash.new
|
144
|
-
wps=Wmap::SiteTracker.
|
148
|
+
wps=Wmap::SiteTracker.instance.known_sites.keys
|
145
149
|
if wps.size > 0
|
146
150
|
Parallel.map(wps, :in_processes => num) { |target|
|
147
151
|
add(target)
|
@@ -197,7 +201,7 @@ class Wmap::WpTracker
|
|
197
201
|
|
198
202
|
# Wordpress detection checkpoint - readme.html
|
199
203
|
def wp_readme?(site)
|
200
|
-
readme_url=site + "
|
204
|
+
readme_url=site + "readme.html"
|
201
205
|
k=Wmap::UrlChecker.new
|
202
206
|
if k.response_code(readme_url) == 200
|
203
207
|
k=nil
|
@@ -216,7 +220,7 @@ class Wmap::WpTracker
|
|
216
220
|
|
217
221
|
# Wordpress detection checkpoint - install.css
|
218
222
|
def wp_css?(site)
|
219
|
-
css_url=site + "
|
223
|
+
css_url=site + "wp-admin/css/install.css"
|
220
224
|
k=Wmap::UrlChecker.new
|
221
225
|
if k.response_code(css_url) == 200
|
222
226
|
k=nil
|
@@ -252,17 +256,79 @@ class Wmap::WpTracker
|
|
252
256
|
return false
|
253
257
|
end
|
254
258
|
|
259
|
+
# Wordpress detection checkpoint - wp-login
|
260
|
+
def wp_login?(url)
|
261
|
+
site=url_2_site(url)
|
262
|
+
login_url=site + "wp-login.php"
|
263
|
+
k=Wmap::UrlChecker.new
|
264
|
+
if k.response_code(login_url) == 200
|
265
|
+
k=nil
|
266
|
+
doc=read_url(login_url)
|
267
|
+
links=doc.css('link')
|
268
|
+
if links.to_s =~ /login.min.css/i
|
269
|
+
return true
|
270
|
+
else
|
271
|
+
return false
|
272
|
+
end
|
273
|
+
end
|
274
|
+
return false
|
275
|
+
end
|
276
|
+
|
277
|
+
# Wordpress detection checkpoint - xml-rpc
|
278
|
+
def wp_rpc?(url)
|
279
|
+
site=url_2_site(url)
|
280
|
+
rpc_url=site + "xmlrpc.php"
|
281
|
+
k=Wmap::UrlChecker.new
|
282
|
+
#puts "res code", k.response_code(rpc_url)
|
283
|
+
if k.response_code(rpc_url) == 405 # method not allowed
|
284
|
+
k=nil
|
285
|
+
return true
|
286
|
+
end
|
287
|
+
return false
|
288
|
+
end
|
289
|
+
|
290
|
+
# Extract the WordPress version
|
255
291
|
def wp_ver(url)
|
256
292
|
if !wp_ver_readme(url).nil?
|
257
293
|
return wp_ver_readme(url)
|
258
294
|
elsif !wp_ver_meta(url).nil?
|
259
295
|
return wp_ver_meta(url)
|
296
|
+
elsif !wp_ver_login(url,"login.min.css").nil?
|
297
|
+
return wp_ver_login(url,"login.min.css")
|
298
|
+
elsif !wp_ver_login(url,"buttons.min.css").nil?
|
299
|
+
return wp_ver_login(url,"buttons.min.css")
|
300
|
+
elsif !wp_ver_login(url,"wp-admin.min.css").nil?
|
301
|
+
return wp_ver_login(url,"wp-admin.min.css")
|
260
302
|
else
|
261
303
|
return nil
|
262
304
|
end
|
263
305
|
end
|
264
306
|
|
265
|
-
# Identify wordpress version through the
|
307
|
+
# Identify wordpress version through the login page
|
308
|
+
def wp_ver_login(url,pattern)
|
309
|
+
puts "Check for #{pattern}" if @verbose
|
310
|
+
site=url_2_site(url)
|
311
|
+
login_url=site + "wp-login.php"
|
312
|
+
k=Wmap::UrlChecker.new
|
313
|
+
#puts "Res code: #{k.response_code(login_url)}" if @verbose
|
314
|
+
if k.response_code(login_url) == 200
|
315
|
+
doc=read_url(login_url)
|
316
|
+
#puts doc.inspect
|
317
|
+
links=doc.css('link')
|
318
|
+
#puts links.inspect if @verbose
|
319
|
+
links.each do |tag|
|
320
|
+
if tag.to_s.include?(pattern)
|
321
|
+
puts tag.to_s if @verbose
|
322
|
+
k=nil
|
323
|
+
return tag.to_s.scan(/[\d+\.]+\d+/).first
|
324
|
+
end
|
325
|
+
end
|
326
|
+
end
|
327
|
+
k=nil
|
328
|
+
return nil
|
329
|
+
end
|
330
|
+
|
331
|
+
# Identify wordpress version through the meta link
|
266
332
|
def wp_ver_meta(url)
|
267
333
|
site=url_2_site(url)
|
268
334
|
k=Wmap::UrlChecker.new
|
@@ -286,13 +352,15 @@ class Wmap::WpTracker
|
|
286
352
|
# Wordpress version detection via - readme.html
|
287
353
|
def wp_ver_readme(url)
|
288
354
|
site=url_2_site(url)
|
289
|
-
readme_url=site + "
|
355
|
+
readme_url=site + "readme.html"
|
290
356
|
k=Wmap::UrlChecker.new
|
357
|
+
puts "Res code: #{k.response_code(readme_url)}" if @verbose
|
291
358
|
if k.response_code(readme_url) == 200
|
292
359
|
k=nil
|
293
360
|
doc=read_url(readme_url)
|
361
|
+
puts doc if @verbose
|
294
362
|
logo=doc.css('h1#logo')[0]
|
295
|
-
|
363
|
+
puts logo.inspect if @verbose
|
296
364
|
return logo.to_s.scan(/[\d+\.]+\d+/).first
|
297
365
|
end
|
298
366
|
k=nil
|