wmap 2.8.2 → 2.8.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/bin/wmap +11 -10
- data/lib/wmap/cidr_tracker.rb +3 -3
- data/lib/wmap/domain_tracker/sub_domain.rb +2 -2
- data/lib/wmap/domain_tracker.rb +3 -3
- data/lib/wmap/host_tracker/primary_host.rb +2 -2
- data/lib/wmap/host_tracker.rb +36 -23
- data/lib/wmap/site_tracker/deactivated_site.rb +1 -1
- data/lib/wmap/site_tracker/wp_tracker.rb +4 -4
- data/lib/wmap/site_tracker.rb +32 -12
- data/lib/wmap/url_checker.rb +3 -3
- data/lib/wmap/url_crawler/adware_tag.rb +2 -2
- data/lib/wmap/url_crawler.rb +1 -1
- data/settings/google_keywords.txt +1 -6
- data/version.txt +2 -2
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6ebcb2cd5065f39431d72629e23c6843927c586c474f0e4e94d4dbb222848cf8
|
4
|
+
data.tar.gz: 0d5b4655157d24e80e015e02e4c3ca6783f41e4bf8385e7b89fb4d0078b96356
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b9b786babd3b056fa637867b00fdd71444ca4e48e73df640d5bfd56f3ed0ce4170babd73f9ea3f0f6e6f310c5f245d80b8857b2d9e2ba1180c0cdb4e1b1cab3a
|
7
|
+
data.tar.gz: b2d875d0a8428e6de815be4d051c94d14412b9643a2eb018e8db8d1024c311b42251a95a0a7818659582478d3a08742a27e1400ac71a4e825f982cb2ed95e455
|
data/README.md
CHANGED
@@ -21,7 +21,7 @@ Note that program is mainly operating on Command Line Interface (CLI). For bette
|
|
21
21
|
|
22
22
|
|
23
23
|
## WMAP in Motion
|
24
|
-
You can try out the complete [demo web app](
|
24
|
+
You can try out the complete [demo web app](https://www.wmap.cloud/) deployed in the DigitalOcean cloud: https://www.wmap.cloud/
|
25
25
|
|
26
26
|
|
27
27
|
## Installation
|
data/bin/wmap
CHANGED
@@ -146,41 +146,42 @@ when nil,[]
|
|
146
146
|
puts "No new site found. There is no change to the site tracking data repository. "
|
147
147
|
else
|
148
148
|
puts "Automatically save the discovery results into the site tracking data repository: "
|
149
|
+
inventory = Wmap::SiteTracker.instance
|
149
150
|
if options[:target] && options[:data_dir]
|
150
151
|
puts "Start the SiteTracker with the optional directory setter. "
|
151
|
-
inventory=Wmap::SiteTracker.instance
|
152
152
|
inventory.data_dir = options[:data_dir]
|
153
|
+
#inventory.verbose = true
|
153
154
|
inventory.sites_file = inventory.data_dir + "/" + "sites"
|
154
155
|
inventory.load_site_stores_from_file(inventory.sites_file)
|
155
156
|
elsif options[:target]
|
156
157
|
puts "Start the SiteTracker. "
|
157
|
-
inventory=Wmap::SiteTracker.instance
|
158
158
|
else
|
159
159
|
abort "Error firing up SiteTracker instance!"
|
160
160
|
end
|
161
161
|
new_sites=inventory.adds(dis_sites.keys-["",nil])
|
162
|
-
|
162
|
+
puts "Newly discovery sties: #{new_sites}"
|
163
|
+
if options[:data_dir]
|
163
164
|
inventory.save!(inventory.sites_file)
|
164
|
-
|
165
|
+
else
|
165
166
|
inventory.save!
|
166
167
|
end
|
167
168
|
inventory=nil
|
168
|
-
puts "Done! New found sites are successfully saved. "
|
169
|
+
puts "Done! New found sites are successfully saved. " unless new_sites.nil? or new_sites.empty?
|
169
170
|
end
|
170
171
|
|
171
172
|
|
172
173
|
# seventh step - update the hosts repository
|
174
|
+
puts "Invoke the HostTracker with optional directory setter."
|
175
|
+
host_tracker = Wmap::HostTracker.instance
|
173
176
|
if options[:target] && options[:data_dir]
|
174
|
-
puts "Invoke the HostTracker with
|
175
|
-
host_tracker = Wmap::HostTracker.instance
|
177
|
+
puts puts "Invoke the HostTracker with options: #{options[:data_dir]}, #{options[:target]}"
|
176
178
|
host_tracker.verbose=options[:verbose]
|
177
179
|
host_tracker.data_dir = options[:data_dir]
|
178
180
|
host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
|
179
181
|
host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
|
180
182
|
elsif options[:target]
|
181
|
-
puts puts "Invoke the HostTracker."
|
182
|
-
host_tracker
|
183
|
-
host_tracker.verbose=options[:verbose]
|
183
|
+
puts puts "Invoke the HostTracker with option: #{options[:target]}."
|
184
|
+
#host_tracker.verbose=options[:verbose]
|
184
185
|
else
|
185
186
|
abort "Error firing up HostTracker instance!"
|
186
187
|
end
|
data/lib/wmap/cidr_tracker.rb
CHANGED
@@ -18,8 +18,8 @@ class Wmap::CidrTracker
|
|
18
18
|
@verbose=params.fetch(:verbose, false)
|
19
19
|
@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
|
20
20
|
Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
|
21
|
-
@cidr_seeds=params.fetch(:cidr_seeds, @data_dir + 'cidrs')
|
22
|
-
File.
|
21
|
+
@cidr_seeds=params.fetch(:cidr_seeds, @data_dir + '/' + 'cidrs')
|
22
|
+
File.new(@cidr_seeds, "w") unless File.exist?(@cidr_seeds)
|
23
23
|
load_cidr_blks_from_file(@cidr_seeds)
|
24
24
|
end
|
25
25
|
|
@@ -48,7 +48,7 @@ class Wmap::CidrTracker
|
|
48
48
|
# 'setter' to load the known CIDR blocks into an instance variable @known_cidr_blks
|
49
49
|
def load_cidr_blks_from_file(file_cidrs=@cidr_seeds)
|
50
50
|
puts "Load the known CIDR seed file: #{file_cidrs}" if @verbose
|
51
|
-
f=File.open(file_cidrs, 'r')
|
51
|
+
f=File.open(file_cidrs, 'r', :encoding => 'UTF-8')
|
52
52
|
f.each do |line|
|
53
53
|
entry=line.chomp.split(',')
|
54
54
|
next unless is_cidr?(entry[0])
|
@@ -27,7 +27,7 @@ class SubDomain < Wmap::DomainTracker
|
|
27
27
|
@max_parallel=params.fetch(:max_parallel, 40)
|
28
28
|
# Hash table to hold the trusted domains
|
29
29
|
@sub_domains_file=params.fetch(:sub_domains_file, @data_dir + 'sub_domains')
|
30
|
-
File.
|
30
|
+
File.new(@sub_domains_file, "w") unless File.exist?(@sub_domains_file)
|
31
31
|
@known_internet_sub_domains=load_domains_from_file(@sub_domains_file) #unless @known_internet_sub_domains.size>0
|
32
32
|
end
|
33
33
|
|
@@ -70,7 +70,7 @@ class SubDomain < Wmap::DomainTracker
|
|
70
70
|
end
|
71
71
|
end
|
72
72
|
@known_internet_sub_domains.merge!(results)
|
73
|
-
puts "Done loading entries."
|
73
|
+
puts "Done loading sub_domain entries."
|
74
74
|
return results
|
75
75
|
else
|
76
76
|
puts "Error: no entry is loaded. Please check your list and try again."
|
data/lib/wmap/domain_tracker.rb
CHANGED
@@ -25,7 +25,7 @@ class Wmap::DomainTracker
|
|
25
25
|
@domains_file=params.fetch(:domains_file, @data_dir+'domains')
|
26
26
|
@max_parallel=params.fetch(:max_parallel, 40)
|
27
27
|
# Hash table to hold the trusted domains
|
28
|
-
File.
|
28
|
+
File.new(@domains_file, "w") unless File.exist?(@domains_file)
|
29
29
|
load_domains_from_file(@domains_file)
|
30
30
|
end
|
31
31
|
|
@@ -169,7 +169,7 @@ class Wmap::DomainTracker
|
|
169
169
|
end
|
170
170
|
end
|
171
171
|
@known_internet_domains.merge!(results)
|
172
|
-
puts "Done loading entries."
|
172
|
+
puts "Done loading domain entries."
|
173
173
|
return results
|
174
174
|
else
|
175
175
|
puts "Error: no entry is loaded. Please check your list and try again."
|
@@ -262,7 +262,7 @@ class Wmap::DomainTracker
|
|
262
262
|
when "Wmap::DomainTracker::SubDomain"
|
263
263
|
return @known_internet_sub_domains.key?(domain)
|
264
264
|
else
|
265
|
-
return
|
265
|
+
return false
|
266
266
|
end
|
267
267
|
rescue => ee
|
268
268
|
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
@@ -21,11 +21,11 @@ module Wmap
|
|
21
21
|
# Initialize the instance variables
|
22
22
|
def initialize (params = {})
|
23
23
|
@verbose=params.fetch(:verbose, false)
|
24
|
-
|
24
|
+
@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../../data/')
|
25
25
|
# Set default instance variables
|
26
26
|
@hosts_file=params.fetch(:hosts_file, @data_dir + 'prime_hosts')
|
27
27
|
# Initialize the instance variables
|
28
|
-
|
28
|
+
File.new(@hosts_file, "w") unless File.exist?(@hosts_file)
|
29
29
|
@known_hosts=load_known_hosts_from_file(@hosts_file)
|
30
30
|
@known_ips=Hash.new
|
31
31
|
de_duplicate
|
data/lib/wmap/host_tracker.rb
CHANGED
@@ -26,16 +26,16 @@ class Wmap::HostTracker
|
|
26
26
|
@hosts_file=params.fetch(:hosts_file, @data_dir + 'hosts')
|
27
27
|
@max_parallel=params.fetch(:max_parallel, 40)
|
28
28
|
# Initialize the instance variables
|
29
|
-
File.
|
30
|
-
load_known_hosts_from_file(@hosts_file)
|
29
|
+
File.new(@hosts_file, "w") unless File.exist?(@hosts_file)
|
30
|
+
@known_hosts=load_known_hosts_from_file(@hosts_file)
|
31
31
|
end
|
32
32
|
|
33
33
|
# Setter to load the known hosts from the local hosts file into a class instance
|
34
34
|
def load_known_hosts_from_file (f_hosts=@hosts_file)
|
35
35
|
puts "Loading local hosts from file: #{f_hosts} ..." if @verbose
|
36
|
-
|
36
|
+
known_hosts=Hash.new
|
37
37
|
@alias = Hash.new
|
38
|
-
File.
|
38
|
+
File.new(f_hosts, "w") unless File.exist?(f_hosts)
|
39
39
|
f=File.open(f_hosts, 'r')
|
40
40
|
f.each do |line|
|
41
41
|
next unless line =~ /\d+\.\d+\.\d+\.\d+/
|
@@ -43,11 +43,11 @@ class Wmap::HostTracker
|
|
43
43
|
key=entry[0].downcase
|
44
44
|
value=entry[1]
|
45
45
|
puts "Loading key value pair: #{key} - #{value}" if @verbose
|
46
|
-
|
47
|
-
|
46
|
+
known_hosts[key] = Hash.new unless known_hosts.key?(key)
|
47
|
+
known_hosts[key]= value
|
48
48
|
# For reverse host lookup
|
49
|
-
|
50
|
-
|
49
|
+
known_hosts[value] = Hash.new unless known_hosts.key?(value)
|
50
|
+
known_hosts[value] = key
|
51
51
|
# Count the number of alias for the recorded IP
|
52
52
|
if @alias.key?(value)
|
53
53
|
@alias[value]+=1
|
@@ -56,11 +56,12 @@ class Wmap::HostTracker
|
|
56
56
|
end
|
57
57
|
end
|
58
58
|
f.close
|
59
|
-
return @known_hosts
|
60
|
-
rescue => ee
|
61
|
-
puts "Exception on method #{__method__}: #{ee}"
|
62
59
|
return known_hosts
|
60
|
+
#rescue => ee
|
61
|
+
# puts "Exception on method #{__method__}: #{ee}"
|
62
|
+
# return known_hosts
|
63
63
|
end
|
64
|
+
alias_method :load, :load_known_hosts_from_file
|
64
65
|
|
65
66
|
# Save the current local hosts hash table into a (random) data repository file
|
66
67
|
def save_known_hosts_to_file!(f_hosts=@hosts_file)
|
@@ -96,30 +97,42 @@ class Wmap::HostTracker
|
|
96
97
|
puts "Exception on method #{__method__}: #{ee}"
|
97
98
|
end
|
98
99
|
|
100
|
+
# determine if host is part of trusted (known) root domains
|
101
|
+
def is_trusted?(host)
|
102
|
+
puts "Determin if host #{host} is part of trusted root domains" if @verbose
|
103
|
+
root=get_domain_root(host)
|
104
|
+
puts "Domain root: #{root}" if @verbose
|
105
|
+
domain_tracker=Wmap::DomainTracker.instance
|
106
|
+
domain_tracker.data_dir=@data_dir
|
107
|
+
domain_tracker.domains_file = domain_tracker.data_dir + "/" + "domains"
|
108
|
+
domain_tracker.load_domains_from_file
|
109
|
+
if domain_tracker.domain_known?(root)
|
110
|
+
domain_tracker=nil
|
111
|
+
return true
|
112
|
+
else
|
113
|
+
domain_tracker=nil
|
114
|
+
return false
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
99
118
|
# Setter to add host entry to the cache once at a time
|
100
119
|
def add(host)
|
101
120
|
puts "Add entry to the local host repository: #{host}"
|
102
121
|
host=host.strip.downcase unless host.nil?
|
122
|
+
root=get_domain_root(host)
|
103
123
|
unless @known_hosts.key?(host)
|
104
124
|
ip=host_2_ip(host)
|
105
125
|
record=Hash.new
|
106
126
|
if is_ip?(ip)
|
107
127
|
# filter host to known domains only
|
108
|
-
|
109
|
-
puts "Domain root: #{root}" if @verbose
|
110
|
-
domain_tracker=Wmap::DomainTracker.instance
|
111
|
-
domain_tracker.data_dir=@data_dir
|
112
|
-
domain_tracker.domains_file = domain_tracker.data_dir + "domains"
|
113
|
-
domain_tracker.load_domains_from_file
|
114
|
-
if domain_tracker.domain_known?(root)
|
115
|
-
domain_tracker=nil
|
128
|
+
if is_trusted?(host)
|
116
129
|
record[host]=ip
|
117
130
|
record[ip]=host
|
118
131
|
puts "Host data repository entry loaded: #{host} <=> #{ip}"
|
119
132
|
# Replace instance with the class variable to avoid potential race condition under parallel engine
|
120
133
|
# add additional logic to update the sub-domain table as well, 02/10/2014
|
121
134
|
sub=get_sub_domain(host)
|
122
|
-
if sub!=
|
135
|
+
if sub!=nil
|
123
136
|
tracker=Wmap::DomainTracker::SubDomain.instance
|
124
137
|
tracker.data_dir=@data_dir
|
125
138
|
tracker.sub_domains_file = tracker.data_dir + "sub_domains"
|
@@ -142,8 +155,8 @@ class Wmap::HostTracker
|
|
142
155
|
else
|
143
156
|
puts "Host is already exist. Skip: #{host}"
|
144
157
|
end
|
145
|
-
rescue => ee
|
146
|
-
|
158
|
+
#rescue => ee
|
159
|
+
# puts "Exception on method #{__method__}: #{ee}" if @verbose
|
147
160
|
end
|
148
161
|
|
149
162
|
# Setter to add host entry to the local hosts in batch (from an array)
|
@@ -164,7 +177,7 @@ class Wmap::HostTracker
|
|
164
177
|
end
|
165
178
|
end
|
166
179
|
@known_hosts.merge!(results)
|
167
|
-
puts "Done loading entries."
|
180
|
+
puts "Done loading host entries."
|
168
181
|
return results
|
169
182
|
else
|
170
183
|
puts "Error: empty list - no entry is loaded. Please check your input list and try again."
|
@@ -27,7 +27,7 @@ class DeactivatedSite < Wmap::SiteTracker
|
|
27
27
|
@sites_file=params.fetch(:sites_file, @data_dir + 'deactivated_sites')
|
28
28
|
@verbose=params.fetch(:verbose, false)
|
29
29
|
# Hash table to hold the site store
|
30
|
-
File.
|
30
|
+
File.new(@sites_file, "w") unless File.exist?(@sites_file)
|
31
31
|
@known_sites=load_site_stores_from_file(@sites_file)
|
32
32
|
end
|
33
33
|
|
@@ -25,13 +25,13 @@ class WpTracker < Wmap::SiteTracker
|
|
25
25
|
@verbose=params.fetch(:verbose, false)
|
26
26
|
@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../../data/')
|
27
27
|
Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
|
28
|
-
|
28
|
+
@sites_wp=params.fetch(:sites_wp, @data_dir+"wp_sites")
|
29
29
|
@http_timeout=params.fetch(:http_timeout, 5000)
|
30
30
|
@max_parallel=params.fetch(:max_parallel, 40)
|
31
31
|
Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
|
32
32
|
@log_file=@data_dir + "wp_checker.log"
|
33
|
-
File.
|
34
|
-
|
33
|
+
File.new(@sites_wp, "w") unless File.exist?(@sites_wp)
|
34
|
+
load_from_file(@sites_wp)
|
35
35
|
end
|
36
36
|
|
37
37
|
# 'setter' to load the known wordpress sites into an instance variable
|
@@ -174,7 +174,7 @@ class WpTracker < Wmap::SiteTracker
|
|
174
174
|
end
|
175
175
|
end
|
176
176
|
@known_wp_sites.merge!(results)
|
177
|
-
puts "Done loading entries."
|
177
|
+
puts "Done loading wp entries."
|
178
178
|
return results
|
179
179
|
else
|
180
180
|
puts "Error: no entry is loaded. Please check your list and try again."
|
data/lib/wmap/site_tracker.rb
CHANGED
@@ -25,7 +25,7 @@ class Wmap::SiteTracker
|
|
25
25
|
@sites_file=params.fetch(:sites_file, @data_dir+'sites')
|
26
26
|
@verbose=params.fetch(:verbose, false)
|
27
27
|
@max_parallel=params.fetch(:max_parallel, 30)
|
28
|
-
File.
|
28
|
+
File.new(@sites_file, "w") unless File.exist?(@sites_file)
|
29
29
|
# Hash table to hold the site store
|
30
30
|
load_site_stores_from_file(@sites_file)
|
31
31
|
end
|
@@ -34,7 +34,7 @@ class Wmap::SiteTracker
|
|
34
34
|
def load_site_stores_from_file (file=@sites_file)
|
35
35
|
puts "Loading the site store data repository from file: #{file} " if @verbose
|
36
36
|
@known_sites=Hash.new
|
37
|
-
File.
|
37
|
+
File.new(file, "w") unless File.exist?(file)
|
38
38
|
f=File.open(file, 'r')
|
39
39
|
f.each do |line|
|
40
40
|
line=line.chomp.strip
|
@@ -77,7 +77,9 @@ class Wmap::SiteTracker
|
|
77
77
|
f.write "# Local site store created by class #{self.class} method #{__method__} at: #{timestamp}\n"
|
78
78
|
f.write "# Website,Primary IP,Port,Hosting Status,Server,Response Code,MD5 Finger-print,Redirection,Timestamp\n"
|
79
79
|
@known_sites.keys.sort.map do |key|
|
80
|
-
|
80
|
+
if is_trusted?(key)
|
81
|
+
f.write "#{key},#{@known_sites[key]['ip']},#{@known_sites[key]['port']},#{@known_sites[key]['status']},#{@known_sites[key]['server']},#{@known_sites[key]['code']},#{@known_sites[key]['md5']},#{@known_sites[key]['redirection']},#{@known_sites[key]['timestamp']}\n"
|
82
|
+
end
|
81
83
|
end
|
82
84
|
f.close
|
83
85
|
puts "site store table is successfully saved: #{file_sites}"
|
@@ -94,6 +96,24 @@ class Wmap::SiteTracker
|
|
94
96
|
puts "Exception on method #{__method__}: #{ee}"
|
95
97
|
end
|
96
98
|
|
99
|
+
# determine site is trusted based on the known domains
|
100
|
+
def is_trusted?(site)
|
101
|
+
trusted=false
|
102
|
+
host=url_2_host(site)
|
103
|
+
root=get_domain_root(host)
|
104
|
+
domain_tracker=Wmap::DomainTracker.instance
|
105
|
+
domain_tracker.data_dir=@data_dir
|
106
|
+
domain_tracker.domains_file=@data_dir + "/" + "domains"
|
107
|
+
File.new(domain_tracker.domains_file, "w") unless File.exist?(domain_tracker.domains_file)
|
108
|
+
domain_tracker.load_domains_from_file(domain_tracker.domains_file)
|
109
|
+
trusted=domain_tracker.domain_known?(root)
|
110
|
+
domain_tracker=nil
|
111
|
+
return trusted
|
112
|
+
rescue => ee
|
113
|
+
puts "Exception on method #{__method__}: #{ee}"
|
114
|
+
return trusted
|
115
|
+
end
|
116
|
+
|
97
117
|
# Setter to add site entry to the cache one at a time
|
98
118
|
def add(site)
|
99
119
|
puts "Add entry to the site store: #{site}"
|
@@ -113,7 +133,7 @@ class Wmap::SiteTracker
|
|
113
133
|
# Additional logic to refresh deactivated site, 02/12/2014
|
114
134
|
deact=Wmap::SiteTracker::DeactivatedSite.instance
|
115
135
|
deact.sites_file=@data_dir + "/" + "deactivated_sites"
|
116
|
-
File.
|
136
|
+
File.new(deact.sites_file, "w") unless File.exist?(deact.sites_file)
|
117
137
|
deact.load_site_stores_from_file
|
118
138
|
# only trust either the domain or IP we know
|
119
139
|
if is_ip?(host)
|
@@ -125,13 +145,17 @@ class Wmap::SiteTracker
|
|
125
145
|
else
|
126
146
|
domain_tracker=Wmap::DomainTracker.instance
|
127
147
|
domain_tracker.domains_file=@data_dir + "/" + "domains"
|
128
|
-
File.
|
148
|
+
File.new(domain_tracker.domains_file, "w") unless File.exist?(domain_tracker.domains_file)
|
129
149
|
domain_tracker.load_domains_from_file(domain_tracker.domains_file)
|
130
150
|
trusted=domain_tracker.domain_known?(root)
|
131
151
|
domain_tracker=nil
|
132
152
|
end
|
133
153
|
end
|
134
154
|
# add record only if trusted
|
155
|
+
host_tracker = Wmap::HostTracker.instance
|
156
|
+
host_tracker.data_dir= @data_dir
|
157
|
+
host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
|
158
|
+
host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
|
135
159
|
if trusted
|
136
160
|
# Add logic to check site status before adding it
|
137
161
|
checker=Wmap::UrlChecker.new(:data_dir=>@data_dir).check(site)
|
@@ -144,10 +168,6 @@ class Wmap::SiteTracker
|
|
144
168
|
raise "Site is currently down. Skip #{site}" if checker['code']==10000
|
145
169
|
end
|
146
170
|
raise "Exception on add method - Fail to resolve the host-name: Host - #{host}, IP - #{ip}. Skip #{site}" unless is_ip?(ip)
|
147
|
-
host_tracker = Wmap::HostTracker.instance
|
148
|
-
host_tracker.data_dir= @data_dir
|
149
|
-
host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
|
150
|
-
host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
|
151
171
|
# Update the local host table when necessary
|
152
172
|
if is_ip?(host)
|
153
173
|
# Case #1: Trusted site contains IP
|
@@ -291,7 +311,7 @@ class Wmap::SiteTracker
|
|
291
311
|
# Additional logic to deactivate the site properly, by moving it to the DeactivatedSite list, 02/07/2014
|
292
312
|
deact=Wmap::SiteTracker::DeactivatedSite.instance
|
293
313
|
deact.sites_file=@data_dir + 'deactivated_sites'
|
294
|
-
File.
|
314
|
+
File.new(deact.sites_file, "w") unless File.exist?(deact.sites_file)
|
295
315
|
site=site.strip.downcase
|
296
316
|
site=url_2_site(site)
|
297
317
|
if @known_sites.key?(site)
|
@@ -341,8 +361,8 @@ class Wmap::SiteTracker
|
|
341
361
|
else
|
342
362
|
puts "Error: no entry is loaded. Please check your list and try again."
|
343
363
|
end
|
344
|
-
|
345
|
-
|
364
|
+
rescue => ee
|
365
|
+
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
346
366
|
end
|
347
367
|
alias_method :dels, :bulk_delete
|
348
368
|
|
data/lib/wmap/url_checker.rb
CHANGED
@@ -88,9 +88,9 @@ class Wmap::UrlChecker
|
|
88
88
|
checker['redirection']=nil
|
89
89
|
checker['timestamp']=timestamp
|
90
90
|
return checker
|
91
|
-
rescue Exception => ee
|
92
|
-
|
93
|
-
|
91
|
+
#rescue Exception => ee
|
92
|
+
# puts "Exception on method #{__method__} for #{url}: #{ee}" # if @verbose
|
93
|
+
# return nil
|
94
94
|
end
|
95
95
|
alias_method :check, :url_worker
|
96
96
|
|
@@ -28,7 +28,7 @@ module Wmap
|
|
28
28
|
file=params.fetch(:signature_file, @signature_file)
|
29
29
|
@tag_signatures=load_sig_from_file(file)
|
30
30
|
@tag_file=params.fetch(:tag_file, @data_dir + 'tag_sites')
|
31
|
-
File.
|
31
|
+
File.new(@tag_file, "w") unless File.exist?(@tag_file)
|
32
32
|
# load the known tag store
|
33
33
|
load_tag_from_file(@tag_file)
|
34
34
|
@landings = Hash.new # cache landing page to reduce redundant browsing
|
@@ -119,7 +119,7 @@ module Wmap
|
|
119
119
|
end
|
120
120
|
end
|
121
121
|
@tag_store.merge!(results)
|
122
|
-
puts "Done loading entries."
|
122
|
+
puts "Done loading adware entries."
|
123
123
|
tags = nil
|
124
124
|
return results
|
125
125
|
else
|
data/lib/wmap/url_crawler.rb
CHANGED
@@ -90,7 +90,7 @@ class Wmap::UrlCrawler
|
|
90
90
|
|
91
91
|
# The worker instance of crawler who perform the labour work
|
92
92
|
def crawl_worker(url0)
|
93
|
-
puts "Please be aware that it may take a while to crawl #{url0}, depending on the site's responsiveness and
|
93
|
+
puts "Please be aware that it may take a while to crawl #{url0}, depending on the site's responsiveness and discovery contents."
|
94
94
|
# Input URL sanity check first
|
95
95
|
if is_url?(url0)
|
96
96
|
host=url_2_host(url0)
|
@@ -1,20 +1,15 @@
|
|
1
1
|
# Default list of Google search keywords (known company legal names, for example) for the service discovery, one entry at per line
|
2
2
|
# Used by the Wmap::GoogleSearchScraper class
|
3
|
+
Google llc
|
3
4
|
|
4
5
|
# Official whole name
|
5
|
-
Penguin Random House
|
6
6
|
|
7
7
|
# 2014 purchase
|
8
|
-
Random House
|
9
8
|
|
10
9
|
# Parent company
|
11
|
-
Bertelsmann
|
12
10
|
|
13
11
|
# Division
|
14
|
-
Crown Books
|
15
12
|
|
16
13
|
# Division
|
17
|
-
Dorling Kindersley
|
18
14
|
|
19
15
|
# Division
|
20
|
-
Penguin Random House Canada
|
data/version.txt
CHANGED
@@ -3,8 +3,8 @@
|
|
3
3
|
###############################################################################
|
4
4
|
package = wmap
|
5
5
|
# wmap version 2.0 == web_discovery version 1.5.3
|
6
|
-
version = 2.8.
|
7
|
-
date =
|
6
|
+
version = 2.8.4
|
7
|
+
date = 2024-07-01
|
8
8
|
|
9
9
|
author = Sam (Yang) Li
|
10
10
|
email = yang.li@owasp.org
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: wmap
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.8.
|
4
|
+
version: 2.8.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sam (Yang) Li
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-07-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: dnsruby
|