wmap 2.4.6 → 2.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2 -2
- data/bin/RHPG +24 -2
- data/bin/distrust +2 -3
- data/bin/trust +1 -1
- data/bin/trusts +1 -1
- data/bin/updateAll +12 -5
- data/bin/wadd +23 -4
- data/bin/wadds +23 -3
- data/bin/wdel +23 -4
- data/bin/wmap +10 -6
- data/bin/wmaps +3 -4
- data/demos/bruter.rb +1 -1
- data/demos/dns_brutes.rb +1 -1
- data/demos/filter_domain.rb +1 -1
- data/demos/filter_known_services.rb +2 -2
- data/demos/filter_prime.rb +1 -1
- data/demos/filter_site.rb +2 -2
- data/demos/filter_siteip.rb +1 -1
- data/demos/filter_url.rb +1 -1
- data/demos/new_fnd.rb +2 -2
- data/demos/site_format.rb +1 -1
- data/demos/whois_domain.rb +2 -2
- data/lib/wmap/dns_bruter.rb +4 -2
- data/lib/wmap/domain_tracker/sub_domain.rb +6 -3
- data/lib/wmap/domain_tracker.rb +3 -2
- data/lib/wmap/host_tracker/primary_host.rb +17 -13
- data/lib/wmap/host_tracker.rb +36 -29
- data/lib/wmap/site_tracker/deactivated_site.rb +2 -2
- data/lib/wmap/site_tracker.rb +25 -12
- data/lib/wmap/url_checker.rb +5 -4
- data/lib/wmap/utils/utils.rb +33 -33
- data/lib/wmap/wp_tracker.rb +75 -7
- data/lib/wmap.rb +12 -11
- data/logs/wmap.log +30 -0
- data/test/domain_tracker_test.rb +5 -5
- data/version.txt +2 -2
- data/wmap.gemspec +1 -1
- metadata +2 -2
data/lib/wmap/host_tracker.rb
CHANGED
@@ -11,7 +11,7 @@ require "singleton" # Implement singleton pattern to avoid race condition under
|
|
11
11
|
|
12
12
|
# Class to handle the local host data repository file where lists of known hosts from discovery and past assessment efforts are stored
|
13
13
|
class Wmap::HostTracker
|
14
|
-
|
14
|
+
include Singleton
|
15
15
|
include Wmap::Utils
|
16
16
|
|
17
17
|
attr_accessor :hosts_file, :max_parallel, :verbose, :data_dir
|
@@ -33,8 +33,8 @@ class Wmap::HostTracker
|
|
33
33
|
|
34
34
|
# Setter to load the known hosts from the local hosts file into a class instance
|
35
35
|
def load_known_hosts_from_file (f_hosts=@file_hosts)
|
36
|
-
|
37
|
-
|
36
|
+
#begin
|
37
|
+
puts "Loading local hosts from file: #{f_hosts} ..." if @verbose
|
38
38
|
known_hosts=Hash.new
|
39
39
|
@alias = Hash.new
|
40
40
|
f=File.open(f_hosts, 'r')
|
@@ -58,16 +58,16 @@ class Wmap::HostTracker
|
|
58
58
|
end
|
59
59
|
f.close
|
60
60
|
return known_hosts
|
61
|
-
rescue => ee
|
62
|
-
|
63
|
-
|
64
|
-
end
|
61
|
+
#rescue => ee
|
62
|
+
# puts "Exception on method #{__method__}: #{ee}"
|
63
|
+
# return known_hosts
|
64
|
+
#end
|
65
65
|
end
|
66
66
|
|
67
67
|
# Save the current local hosts hash table into a (random) data repository file
|
68
68
|
def save_known_hosts_to_file!(f_hosts=@file_hosts)
|
69
|
-
|
70
|
-
|
69
|
+
#begin
|
70
|
+
puts "Saving the local host repository from memory to file: #{f_hosts} ..."
|
71
71
|
timestamp=Time.now
|
72
72
|
f=File.open(f_hosts, 'w')
|
73
73
|
f.write "# local hosts file created by the #{self.class} class #{__method__} method at: #{timestamp}"
|
@@ -76,11 +76,12 @@ class Wmap::HostTracker
|
|
76
76
|
f.write "\n#{key}\t#{@known_hosts[key]}"
|
77
77
|
end
|
78
78
|
end
|
79
|
+
f.write "\n"
|
79
80
|
f.close
|
80
81
|
puts "local host repository is successfully saved to: #{f_hosts}"
|
81
|
-
rescue => ee
|
82
|
-
|
83
|
-
end
|
82
|
+
#rescue => ee
|
83
|
+
# puts "Exception on method #{__method__}: #{ee}"
|
84
|
+
#end
|
84
85
|
end
|
85
86
|
alias_method :save!, :save_known_hosts_to_file!
|
86
87
|
|
@@ -112,7 +113,11 @@ class Wmap::HostTracker
|
|
112
113
|
if is_ip?(ip)
|
113
114
|
# filter host to known domains only
|
114
115
|
root=get_domain_root(host)
|
115
|
-
|
116
|
+
puts "Domain root: #{root}" if @verbose
|
117
|
+
domain_tracker=Wmap::DomainTracker.instance
|
118
|
+
domain_tracker.data_dir=@data_dir
|
119
|
+
if domain_tracker.instance.domain_known?(root)
|
120
|
+
domain_tracker=nil
|
116
121
|
record[host]=ip
|
117
122
|
record[ip]=host
|
118
123
|
puts "Host data repository entry loaded: #{host} <=> #{ip}"
|
@@ -120,7 +125,8 @@ class Wmap::HostTracker
|
|
120
125
|
# add additional logic to update the sub-domain table as well, 02/10/2014
|
121
126
|
sub=get_sub_domain(host)
|
122
127
|
if sub!=root
|
123
|
-
tracker=Wmap::DomainTracker::SubDomain.
|
128
|
+
tracker=Wmap::DomainTracker::SubDomain.instance
|
129
|
+
tracker.data_dir=@data_dir
|
124
130
|
unless tracker.domain_known?(sub)
|
125
131
|
tracker.add(sub)
|
126
132
|
tracker.save!
|
@@ -130,6 +136,7 @@ class Wmap::HostTracker
|
|
130
136
|
@known_hosts.merge!(record)
|
131
137
|
return record
|
132
138
|
else
|
139
|
+
domain_tracker=nil
|
133
140
|
puts "Error - host #{host} has an untrusted internet root domain: #{root}\nPlease update the trusted domain seeds file first if necessary."
|
134
141
|
end
|
135
142
|
else
|
@@ -145,8 +152,8 @@ class Wmap::HostTracker
|
|
145
152
|
|
146
153
|
# Setter to add host entry to the local hosts in batch (from an array)
|
147
154
|
def bulk_add(list, num=@max_parallel)
|
148
|
-
|
149
|
-
|
155
|
+
#begin
|
156
|
+
puts "Add entries to the local host repository: #{list}"
|
150
157
|
results=Hash.new
|
151
158
|
if list.size > 0
|
152
159
|
puts "Start parallel host update processing on:\n #{list}" if @verbose
|
@@ -168,9 +175,9 @@ class Wmap::HostTracker
|
|
168
175
|
puts "Error: empty list - no entry is loaded. Please check your input list and try again."
|
169
176
|
end
|
170
177
|
return results
|
171
|
-
rescue => ee
|
172
|
-
puts "Exception on method #{__method__}: #{ee}"
|
173
|
-
end
|
178
|
+
# rescue => ee
|
179
|
+
# puts "Exception on method #{__method__}: #{ee}"
|
180
|
+
# end
|
174
181
|
end
|
175
182
|
alias_method :adds, :bulk_add
|
176
183
|
|
@@ -242,8 +249,8 @@ class Wmap::HostTracker
|
|
242
249
|
|
243
250
|
# Setter to refresh the entry from the cache one at a time
|
244
251
|
def refresh(host)
|
245
|
-
|
246
|
-
|
252
|
+
#begin
|
253
|
+
puts "Refresh the local host repository for host: #{host} "
|
247
254
|
host=host.strip.downcase
|
248
255
|
if @known_hosts.key?(host)
|
249
256
|
old_ip=@known_hosts[host]
|
@@ -266,15 +273,15 @@ class Wmap::HostTracker
|
|
266
273
|
else
|
267
274
|
puts "Error entry non exist: #{host}"
|
268
275
|
end
|
269
|
-
rescue => ee
|
270
|
-
|
271
|
-
end
|
276
|
+
#rescue => ee
|
277
|
+
# puts "Exception on method #{__method__}: #{ee}"
|
278
|
+
#end
|
272
279
|
end
|
273
280
|
|
274
281
|
# Refresh all the entries in the local hosts by querying the Internet
|
275
282
|
def refresh_all
|
276
283
|
puts "Refresh all the entries in the local host repository in one shot."
|
277
|
-
begin
|
284
|
+
#begin
|
278
285
|
changes=Hash.new
|
279
286
|
hosts=@known_hosts.keys
|
280
287
|
@known_hosts=Hash.new
|
@@ -290,9 +297,9 @@ class Wmap::HostTracker
|
|
290
297
|
#changes.map { |x| puts x }
|
291
298
|
puts "Done refreshing the local hosts."
|
292
299
|
return changes
|
293
|
-
rescue => ee
|
294
|
-
|
295
|
-
end
|
300
|
+
#rescue => ee
|
301
|
+
# puts "Exception on method #{__method__}: #{ee}"
|
302
|
+
#end
|
296
303
|
end
|
297
304
|
|
298
305
|
# Extract known root domains from the local host repository @known_hosts
|
@@ -439,7 +446,7 @@ class Wmap::HostTracker
|
|
439
446
|
sub = get_subdomain(hostname)
|
440
447
|
subs.push(sub) unless sub.nil?
|
441
448
|
end
|
442
|
-
subs.uniq!.sort!
|
449
|
+
subs.uniq!.sort! unless subs.empty?
|
443
450
|
puts "Found sub domains: #{subs}" if @verbose
|
444
451
|
return subs
|
445
452
|
rescue Exception => ee
|
@@ -5,7 +5,7 @@
|
|
5
5
|
#
|
6
6
|
# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
|
7
7
|
#++
|
8
|
-
|
8
|
+
require "singleton"
|
9
9
|
|
10
10
|
|
11
11
|
# Class to trace de-activated site. This is need for basic state tracking for our sites.
|
@@ -15,7 +15,7 @@ class SiteTracker
|
|
15
15
|
|
16
16
|
class DeactivatedSite < Wmap::SiteTracker
|
17
17
|
include Wmap::Utils
|
18
|
-
|
18
|
+
include Singleton
|
19
19
|
|
20
20
|
attr_accessor :sites_file, :known_sites, :verbose, :data_dir
|
21
21
|
|
data/lib/wmap/site_tracker.rb
CHANGED
@@ -6,14 +6,14 @@
|
|
6
6
|
# Copyright (c) 2012-2015 Yang Li <yang.li@owasp.org>
|
7
7
|
#++
|
8
8
|
require "parallel"
|
9
|
-
|
9
|
+
require "singleton"
|
10
10
|
require "nokogiri"
|
11
11
|
|
12
12
|
|
13
13
|
# Main class to automatically track the site inventory
|
14
14
|
class Wmap::SiteTracker
|
15
15
|
include Wmap::Utils
|
16
|
-
|
16
|
+
include Singleton
|
17
17
|
|
18
18
|
attr_accessor :sites_file, :max_parallel, :verbose, :data_dir
|
19
19
|
attr_reader :known_sites
|
@@ -116,7 +116,8 @@ class Wmap::SiteTracker
|
|
116
116
|
host=url_2_host(site)
|
117
117
|
ip=host_2_ip(host)
|
118
118
|
# Additional logic to refresh deactivated site, 02/12/2014
|
119
|
-
deact=Wmap::SiteTracker::DeactivatedSite.
|
119
|
+
deact=Wmap::SiteTracker::DeactivatedSite.instance
|
120
|
+
deact.data_dir=@data_dir
|
120
121
|
# only trust either the domain or IP we know
|
121
122
|
if is_ip?(host)
|
122
123
|
trusted=Wmap::CidrTracker.new(:data_dir=>@data_dir).ip_trusted?(ip)
|
@@ -125,7 +126,10 @@ class Wmap::SiteTracker
|
|
125
126
|
if root.nil?
|
126
127
|
raise "Invalid web site format. Please check your record again."
|
127
128
|
else
|
128
|
-
|
129
|
+
domain_tracker=Wmap::DomainTracker.instance
|
130
|
+
domain_tracker.data_dir=@data_dir
|
131
|
+
trusted=domain_tracker.domain_known?(root)
|
132
|
+
domain_tracker=nil
|
129
133
|
end
|
130
134
|
end
|
131
135
|
# add record only if trusted
|
@@ -140,7 +144,8 @@ class Wmap::SiteTracker
|
|
140
144
|
raise "Site is currently down. Skip #{site}" if checker['code']==10000
|
141
145
|
end
|
142
146
|
raise "Exception on add method - Fail to resolve the host-name: Host - #{host}, IP - #{ip}. Skip #{site}" unless is_ip?(ip)
|
143
|
-
my_tracker = Wmap::HostTracker.
|
147
|
+
my_tracker = Wmap::HostTracker.instance
|
148
|
+
my_tracker.data_dir=@data_dir
|
144
149
|
# Update the local host table when necessary
|
145
150
|
if is_ip?(host)
|
146
151
|
# Case #1: Trusted site contains IP
|
@@ -156,12 +161,15 @@ class Wmap::SiteTracker
|
|
156
161
|
host1=ip_2_host(host)
|
157
162
|
puts "host1: #{host1}" if @verbose
|
158
163
|
if is_fqdn?(host1)
|
159
|
-
|
164
|
+
host_tracker=Wmap::HostTracker.instance
|
165
|
+
host_tracker.data_dir=@data_dir
|
166
|
+
if host_tracker.domain_known?(host1)
|
160
167
|
# replace IP with host-name only if domain root is known
|
161
168
|
puts "Host found from the Internet reverse DNS lookup for #{ip}: #{host1}" if @verbose
|
162
169
|
host=host1
|
163
170
|
site.sub!(/\d+\.\d+\.\d+\.\d+/,host)
|
164
171
|
end
|
172
|
+
host_tracker=nil
|
165
173
|
end
|
166
174
|
end
|
167
175
|
# Adding site for Case #1
|
@@ -288,7 +296,8 @@ class Wmap::SiteTracker
|
|
288
296
|
puts "Remove entry from the site store: #{site} " if @verbose
|
289
297
|
begin
|
290
298
|
# Additional logic to deactivate the site properly, by moving it to the DeactivatedSite list, 02/07/2014
|
291
|
-
deact=Wmap::SiteTracker::DeactivatedSite.
|
299
|
+
deact=Wmap::SiteTracker::DeactivatedSite.instance
|
300
|
+
deact.data_dir=@data_dir
|
292
301
|
site=site.strip.downcase
|
293
302
|
site=url_2_site(site)
|
294
303
|
if @known_sites.key?(site)
|
@@ -574,10 +583,11 @@ class Wmap::SiteTracker
|
|
574
583
|
def get_uniq_sites
|
575
584
|
puts "Getter to retrieve unique sites containing unique IP:PORT key identifier." if @verbose
|
576
585
|
begin
|
577
|
-
#primary_host_tracker=Wmap::HostTracker::PrimaryHost.
|
586
|
+
#primary_host_tracker=Wmap::HostTracker::PrimaryHost.instance
|
578
587
|
sites=Hash.new
|
579
588
|
#uniqueness=Hash.new
|
580
|
-
my_tracker=Wmap::HostTracker.
|
589
|
+
my_tracker=Wmap::HostTracker.instance
|
590
|
+
my_tracker.data_dir=@data_dir
|
581
591
|
@known_sites.keys.map do |key|
|
582
592
|
port=url_2_port(key).to_s
|
583
593
|
host=url_2_host(key)
|
@@ -677,7 +687,8 @@ class Wmap::SiteTracker
|
|
677
687
|
begin
|
678
688
|
updates=Array.new
|
679
689
|
sites=get_ip_sites
|
680
|
-
my_tracker=Wmap::HostTracker.
|
690
|
+
my_tracker=Wmap::HostTracker.instance
|
691
|
+
my_tracker.data_dir=@data_dir
|
681
692
|
sites.map do |site|
|
682
693
|
puts "Work on resolve the IP site: #{site}" if @verbose
|
683
694
|
ip=url_2_host(site)
|
@@ -837,8 +848,10 @@ class Wmap::SiteTracker
|
|
837
848
|
def get_prim_uniq_sites
|
838
849
|
puts "Retrieve and prime unique sites in the site store. " if @verbose
|
839
850
|
#begin
|
840
|
-
host_tracker=Wmap::HostTracker.
|
841
|
-
|
851
|
+
host_tracker=Wmap::HostTracker.instance
|
852
|
+
host_tracker.data_dir=@data_dir
|
853
|
+
primary_host_tracker=Wmap::HostTracker::PrimaryHost.instance
|
854
|
+
primary_host_tracker.data_dir=@data_dir
|
842
855
|
# Step 1. Retrieve the unique site list first
|
843
856
|
sites=get_uniq_sites
|
844
857
|
prim_uniq_sites=Array.new
|
data/lib/wmap/url_checker.rb
CHANGED
@@ -176,9 +176,9 @@ class Wmap::UrlChecker
|
|
176
176
|
|
177
177
|
# Test the URL / site and return the redirection location (3xx response code only)
|
178
178
|
def redirect_location (url)
|
179
|
-
puts "Test the redirection location for the url: #{url}" if @verbose
|
180
|
-
location=""
|
181
179
|
begin
|
180
|
+
puts "Test the redirection location for the url: #{url}" if @verbose
|
181
|
+
location=""
|
182
182
|
raise "Invalid url: #{url}" unless is_url?(url)
|
183
183
|
url=url.strip.downcase
|
184
184
|
timeo = @http_timeout/1000.0
|
@@ -196,6 +196,7 @@ class Wmap::UrlChecker
|
|
196
196
|
end
|
197
197
|
request = Net::HTTP::Get.new(uri.request_uri)
|
198
198
|
response = http.request(request)
|
199
|
+
puts "Response: #{response}" if @verbose
|
199
200
|
case response
|
200
201
|
when Net::HTTPRedirection then
|
201
202
|
location = response['location']
|
@@ -213,9 +214,9 @@ class Wmap::UrlChecker
|
|
213
214
|
|
214
215
|
# Test the URL / site and return the web server type from the HTTP header "server" field
|
215
216
|
def get_server_header (url)
|
216
|
-
puts "Retrieve the server header field from the url: #{url}" if @verbose
|
217
|
-
server=String.new
|
218
217
|
begin
|
218
|
+
puts "Retrieve the server header field from the url: #{url}" if @verbose
|
219
|
+
server=String.new
|
219
220
|
raise "Invalid url: #{url}" unless is_url?(url)
|
220
221
|
url=url.strip.downcase
|
221
222
|
timeo = @http_timeout/1000.0
|
data/lib/wmap/utils/utils.rb
CHANGED
@@ -11,12 +11,12 @@ require "netaddr"
|
|
11
11
|
|
12
12
|
# Main utility module to provide the common functions across different classes
|
13
13
|
module Wmap
|
14
|
-
module Utils
|
14
|
+
module Utils
|
15
15
|
include Wmap::Utils::DomainRoot
|
16
16
|
include Wmap::Utils::UrlMagic
|
17
17
|
include Wmap::Utils::Logger
|
18
18
|
extend self
|
19
|
-
|
19
|
+
|
20
20
|
# Load entries from a text file and return an array
|
21
21
|
def file_2_list(f,lc=true)
|
22
22
|
puts "Loading records from file: #{f}" if @verbose
|
@@ -27,7 +27,7 @@ module Wmap
|
|
27
27
|
line=line.chomp.strip
|
28
28
|
next if line.nil?
|
29
29
|
next if line.empty?
|
30
|
-
next if line =~ /^\s*#/
|
30
|
+
next if line =~ /^\s*#/
|
31
31
|
line=line.downcase if lc==true
|
32
32
|
list.push(line.chomp.strip)
|
33
33
|
end
|
@@ -55,7 +55,7 @@ module Wmap
|
|
55
55
|
return nil
|
56
56
|
end
|
57
57
|
end
|
58
|
-
|
58
|
+
|
59
59
|
# Load entries from a text file and return a hash
|
60
60
|
def file_2_hash(f,lc=true)
|
61
61
|
puts "Loading records from file: #{f}" if @verbose
|
@@ -67,7 +67,7 @@ module Wmap
|
|
67
67
|
next if line.nil?
|
68
68
|
next if line.empty?
|
69
69
|
line=line.downcase if lc==true
|
70
|
-
next if line =~ /^\s*#/
|
70
|
+
next if line =~ /^\s*#/
|
71
71
|
hs[line]=true unless hs.key?(line)
|
72
72
|
end
|
73
73
|
file.close
|
@@ -77,7 +77,7 @@ module Wmap
|
|
77
77
|
return nil
|
78
78
|
end
|
79
79
|
end
|
80
|
-
|
80
|
+
|
81
81
|
# Query the name-server to see if the dns record is still valid
|
82
82
|
def valid_dns_record? (hostname)
|
83
83
|
puts "Validate the hostname record: #{hostname}" if @verbose
|
@@ -99,7 +99,7 @@ module Wmap
|
|
99
99
|
|
100
100
|
# Test the DNS server if zone transfer is allowed. If allowed, save the found hosts into the class variable.
|
101
101
|
def zone_transferable?(domain)
|
102
|
-
puts "Check if the domain allows free zone transfer: #{domain}"
|
102
|
+
puts "Check if the domain allows free zone transfer: #{domain}"
|
103
103
|
begin
|
104
104
|
transferable=false
|
105
105
|
domain=domain.downcase
|
@@ -108,32 +108,32 @@ module Wmap
|
|
108
108
|
puts "Retrieved name servers: #{nameservers}" if @verbose
|
109
109
|
nameservers.each do |nssrv|
|
110
110
|
begin
|
111
|
-
puts "Attempt zone transfer on name server: #{nssrv}"
|
111
|
+
puts "Attempt zone transfer on name server: #{nssrv}"
|
112
112
|
if nssrv.nil?
|
113
113
|
abort "Method input variable error: no name server found!" if @verbose
|
114
114
|
next
|
115
|
-
end
|
115
|
+
end
|
116
116
|
zt = Dnsruby::ZoneTransfer.new
|
117
|
-
zt.server=(nssrv) if nssrv!=""
|
117
|
+
zt.server=(nssrv) if nssrv!=""
|
118
118
|
records = zt.transfer(domain)
|
119
119
|
if records==nil
|
120
|
-
puts "#{domain} zone transfer is not allowed on name server: #{nssrv}"
|
120
|
+
puts "#{domain} zone transfer is not allowed on name server: #{nssrv}"
|
121
121
|
next
|
122
122
|
else
|
123
123
|
puts "#{domain} zone transfer is allowed!"
|
124
124
|
transferable=true
|
125
125
|
end
|
126
126
|
rescue Exception=>ee
|
127
|
-
puts "Exception on method zone_transferable? for domain #{domain}: #{ee}"
|
127
|
+
puts "Exception on method zone_transferable? for domain #{domain}: #{ee}"
|
128
128
|
end
|
129
129
|
end
|
130
130
|
return transferable
|
131
131
|
rescue Exception => ee
|
132
132
|
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
133
|
-
return
|
134
|
-
end
|
135
|
-
end
|
136
|
-
|
133
|
+
return false
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
137
|
# Test if it's a legitimate IP4 address
|
138
138
|
def is_ip? (ip)
|
139
139
|
puts "Validate the IP format is valid: #{ip}" if @verbose
|
@@ -155,8 +155,8 @@ module Wmap
|
|
155
155
|
end
|
156
156
|
end
|
157
157
|
alias_method :is_valid_ip?, :is_ip?
|
158
|
-
|
159
|
-
# Simple test a host string format. Return true if it contains a valid internet domain sub-string. Note: Don't be confused with another method 'valid_dns_record?', which is a stricter and time-consuming test on the DNS server for a resolvable internet host.
|
158
|
+
|
159
|
+
# Simple test a host string format. Return true if it contains a valid internet domain sub-string. Note: Don't be confused with another method 'valid_dns_record?', which is a stricter and time-consuming test on the DNS server for a resolvable internet host.
|
160
160
|
def is_fqdn? (host)
|
161
161
|
puts "Validate the host-name format is valid: #{host}" if @verbose
|
162
162
|
begin
|
@@ -202,17 +202,17 @@ module Wmap
|
|
202
202
|
return false
|
203
203
|
end
|
204
204
|
end
|
205
|
-
|
205
|
+
|
206
206
|
# Sort an array of IPs in the ascendant order
|
207
207
|
def sort_ips (ips)
|
208
208
|
begin
|
209
209
|
"Sort the list of IP address in the ascendant order: #{ips}" if @verbose
|
210
210
|
return NetAddr.sort(ips)
|
211
|
-
rescue => ee
|
211
|
+
rescue => ee
|
212
212
|
puts "Exception on method sort_ips for IPs #{ips}: #{ee}" # if @verbose
|
213
|
-
end
|
213
|
+
end
|
214
214
|
end
|
215
|
-
|
215
|
+
|
216
216
|
# Perform the DNS query on a hostname over the Internet. Return the resolved IP(s) in an array
|
217
217
|
def host_2_ips (hostname)
|
218
218
|
begin
|
@@ -233,7 +233,7 @@ module Wmap
|
|
233
233
|
puts "Exception on method host_2_ips for host #{hostname}: #{ee}" if @verbose
|
234
234
|
return nil
|
235
235
|
end
|
236
|
-
end
|
236
|
+
end
|
237
237
|
alias_method :ns_lookup, :host_2_ips
|
238
238
|
|
239
239
|
# Perform DNS query on a hostname. Return the first resolved IP as a string
|
@@ -258,8 +258,8 @@ module Wmap
|
|
258
258
|
puts "Exception on method host_2_ip for host #{hostname}: #{ee}" if @verbose
|
259
259
|
return nil
|
260
260
|
end
|
261
|
-
end
|
262
|
-
|
261
|
+
end
|
262
|
+
|
263
263
|
# Retrieve a list of the authoritative name servers from the Internet whois data repository for the host / subdomain / domain
|
264
264
|
def get_nameservers (host)
|
265
265
|
puts "Retrieve a list of authoritative name server for: #{host}" if @verbose
|
@@ -269,7 +269,7 @@ module Wmap
|
|
269
269
|
ns = w.query(domain).nameservers.map! { |x| x.name }
|
270
270
|
if ns.empty?
|
271
271
|
puts "No name server found for domain root: #{domain}" if @verbose
|
272
|
-
return nil
|
272
|
+
return nil
|
273
273
|
else
|
274
274
|
return ns
|
275
275
|
end
|
@@ -288,7 +288,7 @@ module Wmap
|
|
288
288
|
ns = w.query(domain).nameservers.map! { |x| x.name }
|
289
289
|
if ns.empty?
|
290
290
|
puts "No name server found for domain root: #{domain}" if @verbose
|
291
|
-
return nil
|
291
|
+
return nil
|
292
292
|
else
|
293
293
|
return ns.first
|
294
294
|
end
|
@@ -298,21 +298,21 @@ module Wmap
|
|
298
298
|
end
|
299
299
|
end
|
300
300
|
alias_method :get_ns, :get_nameserver
|
301
|
-
|
301
|
+
|
302
302
|
# Perform reverse dns lookup for an IP. Return the found 'hostname' if found, or the original IP if not
|
303
303
|
def reverse_dns_lookup (ip)
|
304
304
|
puts "Retrieve the hostname by the reverse DNS lookup on IP: #{ip}"
|
305
305
|
hostname = ip
|
306
|
-
begin
|
306
|
+
begin
|
307
307
|
hostname = Socket.gethostbyaddr(ip.split('.').collect{ |x| x.to_i}.pack("CCCC"))[0]
|
308
308
|
return hostname.downcase
|
309
309
|
rescue => ee
|
310
310
|
puts "Exception on method reverse_dns_lookup: #{ee}" if @verbose
|
311
311
|
return hostname
|
312
|
-
end
|
312
|
+
end
|
313
313
|
end
|
314
314
|
alias_method :ip_2_host, :reverse_dns_lookup
|
315
|
-
|
315
|
+
|
316
316
|
# Convert a CIDR to a list of IPs: Input is a CIDR expression such as '192.168.1.1/30', output is an array of IPs
|
317
317
|
def cidr_2_ips (cidr)
|
318
318
|
puts "Method to convert a CIDR block into a list of IPs: #{cidr}" if @verbose
|
@@ -324,10 +324,10 @@ module Wmap
|
|
324
324
|
rescue => ee
|
325
325
|
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
326
326
|
return nil
|
327
|
-
end
|
327
|
+
end
|
328
328
|
end
|
329
329
|
|
330
330
|
|
331
|
-
|
331
|
+
|
332
332
|
end
|
333
333
|
end
|
data/lib/wmap/wp_tracker.rb
CHANGED
@@ -87,7 +87,7 @@ class Wmap::WpTracker
|
|
87
87
|
end
|
88
88
|
end
|
89
89
|
f.close
|
90
|
-
puts "
|
90
|
+
puts "WordPress site cache table is successfully saved: #{file_wps}"
|
91
91
|
rescue => ee
|
92
92
|
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
93
93
|
end
|
@@ -127,6 +127,10 @@ class Wmap::WpTracker
|
|
127
127
|
found=true
|
128
128
|
elsif wp_meta?(site)
|
129
129
|
found=true
|
130
|
+
elsif wp_login?(site)
|
131
|
+
found=true
|
132
|
+
elsif wp_rpc?(site)
|
133
|
+
found=true
|
130
134
|
else
|
131
135
|
found=false
|
132
136
|
end
|
@@ -141,7 +145,7 @@ class Wmap::WpTracker
|
|
141
145
|
#begin
|
142
146
|
puts "Add entries to the local cache table from site tracker: " if @verbose
|
143
147
|
results=Hash.new
|
144
|
-
wps=Wmap::SiteTracker.
|
148
|
+
wps=Wmap::SiteTracker.instance.known_sites.keys
|
145
149
|
if wps.size > 0
|
146
150
|
Parallel.map(wps, :in_processes => num) { |target|
|
147
151
|
add(target)
|
@@ -197,7 +201,7 @@ class Wmap::WpTracker
|
|
197
201
|
|
198
202
|
# Wordpress detection checkpoint - readme.html
|
199
203
|
def wp_readme?(site)
|
200
|
-
readme_url=site + "
|
204
|
+
readme_url=site + "readme.html"
|
201
205
|
k=Wmap::UrlChecker.new
|
202
206
|
if k.response_code(readme_url) == 200
|
203
207
|
k=nil
|
@@ -216,7 +220,7 @@ class Wmap::WpTracker
|
|
216
220
|
|
217
221
|
# Wordpress detection checkpoint - install.css
|
218
222
|
def wp_css?(site)
|
219
|
-
css_url=site + "
|
223
|
+
css_url=site + "wp-admin/css/install.css"
|
220
224
|
k=Wmap::UrlChecker.new
|
221
225
|
if k.response_code(css_url) == 200
|
222
226
|
k=nil
|
@@ -252,17 +256,79 @@ class Wmap::WpTracker
|
|
252
256
|
return false
|
253
257
|
end
|
254
258
|
|
259
|
+
# Wordpress detection checkpoint - wp-login
|
260
|
+
def wp_login?(url)
|
261
|
+
site=url_2_site(url)
|
262
|
+
login_url=site + "wp-login.php"
|
263
|
+
k=Wmap::UrlChecker.new
|
264
|
+
if k.response_code(login_url) == 200
|
265
|
+
k=nil
|
266
|
+
doc=read_url(login_url)
|
267
|
+
links=doc.css('link')
|
268
|
+
if links.to_s =~ /login.min.css/i
|
269
|
+
return true
|
270
|
+
else
|
271
|
+
return false
|
272
|
+
end
|
273
|
+
end
|
274
|
+
return false
|
275
|
+
end
|
276
|
+
|
277
|
+
# Wordpress detection checkpoint - xml-rpc
|
278
|
+
def wp_rpc?(url)
|
279
|
+
site=url_2_site(url)
|
280
|
+
rpc_url=site + "xmlrpc.php"
|
281
|
+
k=Wmap::UrlChecker.new
|
282
|
+
#puts "res code", k.response_code(rpc_url)
|
283
|
+
if k.response_code(rpc_url) == 405 # method not allowed
|
284
|
+
k=nil
|
285
|
+
return true
|
286
|
+
end
|
287
|
+
return false
|
288
|
+
end
|
289
|
+
|
290
|
+
# Extract the WordPress version
|
255
291
|
def wp_ver(url)
|
256
292
|
if !wp_ver_readme(url).nil?
|
257
293
|
return wp_ver_readme(url)
|
258
294
|
elsif !wp_ver_meta(url).nil?
|
259
295
|
return wp_ver_meta(url)
|
296
|
+
elsif !wp_ver_login(url,"login.min.css").nil?
|
297
|
+
return wp_ver_login(url,"login.min.css")
|
298
|
+
elsif !wp_ver_login(url,"buttons.min.css").nil?
|
299
|
+
return wp_ver_login(url,"buttons.min.css")
|
300
|
+
elsif !wp_ver_login(url,"wp-admin.min.css").nil?
|
301
|
+
return wp_ver_login(url,"wp-admin.min.css")
|
260
302
|
else
|
261
303
|
return nil
|
262
304
|
end
|
263
305
|
end
|
264
306
|
|
265
|
-
# Identify wordpress version through the
|
307
|
+
# Identify wordpress version through the login page
|
308
|
+
def wp_ver_login(url,pattern)
|
309
|
+
puts "Check for #{pattern}" if @verbose
|
310
|
+
site=url_2_site(url)
|
311
|
+
login_url=site + "wp-login.php"
|
312
|
+
k=Wmap::UrlChecker.new
|
313
|
+
#puts "Res code: #{k.response_code(login_url)}" if @verbose
|
314
|
+
if k.response_code(login_url) == 200
|
315
|
+
doc=read_url(login_url)
|
316
|
+
#puts doc.inspect
|
317
|
+
links=doc.css('link')
|
318
|
+
#puts links.inspect if @verbose
|
319
|
+
links.each do |tag|
|
320
|
+
if tag.to_s.include?(pattern)
|
321
|
+
puts tag.to_s if @verbose
|
322
|
+
k=nil
|
323
|
+
return tag.to_s.scan(/[\d+\.]+\d+/).first
|
324
|
+
end
|
325
|
+
end
|
326
|
+
end
|
327
|
+
k=nil
|
328
|
+
return nil
|
329
|
+
end
|
330
|
+
|
331
|
+
# Identify wordpress version through the meta link
|
266
332
|
def wp_ver_meta(url)
|
267
333
|
site=url_2_site(url)
|
268
334
|
k=Wmap::UrlChecker.new
|
@@ -286,13 +352,15 @@ class Wmap::WpTracker
|
|
286
352
|
# Wordpress version detection via - readme.html
|
287
353
|
def wp_ver_readme(url)
|
288
354
|
site=url_2_site(url)
|
289
|
-
readme_url=site + "
|
355
|
+
readme_url=site + "readme.html"
|
290
356
|
k=Wmap::UrlChecker.new
|
357
|
+
puts "Res code: #{k.response_code(readme_url)}" if @verbose
|
291
358
|
if k.response_code(readme_url) == 200
|
292
359
|
k=nil
|
293
360
|
doc=read_url(readme_url)
|
361
|
+
puts doc if @verbose
|
294
362
|
logo=doc.css('h1#logo')[0]
|
295
|
-
|
363
|
+
puts logo.inspect if @verbose
|
296
364
|
return logo.to_s.scan(/[\d+\.]+\d+/).first
|
297
365
|
end
|
298
366
|
k=nil
|