wmap 2.8.2 → 2.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/bin/wmap +11 -10
- data/lib/wmap/cidr_tracker.rb +3 -3
- data/lib/wmap/domain_tracker/sub_domain.rb +2 -2
- data/lib/wmap/domain_tracker.rb +3 -3
- data/lib/wmap/host_tracker/primary_host.rb +2 -2
- data/lib/wmap/host_tracker.rb +36 -23
- data/lib/wmap/site_tracker/deactivated_site.rb +1 -1
- data/lib/wmap/site_tracker/wp_tracker.rb +4 -4
- data/lib/wmap/site_tracker.rb +32 -12
- data/lib/wmap/url_checker.rb +3 -3
- data/lib/wmap/url_crawler/adware_tag.rb +2 -2
- data/lib/wmap/url_crawler.rb +1 -1
- data/settings/google_keywords.txt +1 -6
- data/version.txt +2 -2
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6ebcb2cd5065f39431d72629e23c6843927c586c474f0e4e94d4dbb222848cf8
|
4
|
+
data.tar.gz: 0d5b4655157d24e80e015e02e4c3ca6783f41e4bf8385e7b89fb4d0078b96356
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b9b786babd3b056fa637867b00fdd71444ca4e48e73df640d5bfd56f3ed0ce4170babd73f9ea3f0f6e6f310c5f245d80b8857b2d9e2ba1180c0cdb4e1b1cab3a
|
7
|
+
data.tar.gz: b2d875d0a8428e6de815be4d051c94d14412b9643a2eb018e8db8d1024c311b42251a95a0a7818659582478d3a08742a27e1400ac71a4e825f982cb2ed95e455
|
data/README.md
CHANGED
@@ -21,7 +21,7 @@ Note that program is mainly operating on Command Line Interface (CLI). For bette
|
|
21
21
|
|
22
22
|
|
23
23
|
## WMAP in Motion
|
24
|
-
You can try out the complete [demo web app](
|
24
|
+
You can try out the complete [demo web app](https://www.wmap.cloud/) deployed in the DigitalOcean cloud: https://www.wmap.cloud/
|
25
25
|
|
26
26
|
|
27
27
|
## Installation
|
data/bin/wmap
CHANGED
@@ -146,41 +146,42 @@ when nil,[]
|
|
146
146
|
puts "No new site found. There is no change to the site tracking data repository. "
|
147
147
|
else
|
148
148
|
puts "Automatically save the discovery results into the site tracking data repository: "
|
149
|
+
inventory = Wmap::SiteTracker.instance
|
149
150
|
if options[:target] && options[:data_dir]
|
150
151
|
puts "Start the SiteTracker with the optional directory setter. "
|
151
|
-
inventory=Wmap::SiteTracker.instance
|
152
152
|
inventory.data_dir = options[:data_dir]
|
153
|
+
#inventory.verbose = true
|
153
154
|
inventory.sites_file = inventory.data_dir + "/" + "sites"
|
154
155
|
inventory.load_site_stores_from_file(inventory.sites_file)
|
155
156
|
elsif options[:target]
|
156
157
|
puts "Start the SiteTracker. "
|
157
|
-
inventory=Wmap::SiteTracker.instance
|
158
158
|
else
|
159
159
|
abort "Error firing up SiteTracker instance!"
|
160
160
|
end
|
161
161
|
new_sites=inventory.adds(dis_sites.keys-["",nil])
|
162
|
-
|
162
|
+
puts "Newly discovery sties: #{new_sites}"
|
163
|
+
if options[:data_dir]
|
163
164
|
inventory.save!(inventory.sites_file)
|
164
|
-
|
165
|
+
else
|
165
166
|
inventory.save!
|
166
167
|
end
|
167
168
|
inventory=nil
|
168
|
-
puts "Done! New found sites are successfully saved. "
|
169
|
+
puts "Done! New found sites are successfully saved. " unless new_sites.nil? or new_sites.empty?
|
169
170
|
end
|
170
171
|
|
171
172
|
|
172
173
|
# seventh step - update the hosts repository
|
174
|
+
puts "Invoke the HostTracker with optional directory setter."
|
175
|
+
host_tracker = Wmap::HostTracker.instance
|
173
176
|
if options[:target] && options[:data_dir]
|
174
|
-
puts "Invoke the HostTracker with
|
175
|
-
host_tracker = Wmap::HostTracker.instance
|
177
|
+
puts puts "Invoke the HostTracker with options: #{options[:data_dir]}, #{options[:target]}"
|
176
178
|
host_tracker.verbose=options[:verbose]
|
177
179
|
host_tracker.data_dir = options[:data_dir]
|
178
180
|
host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
|
179
181
|
host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
|
180
182
|
elsif options[:target]
|
181
|
-
puts puts "Invoke the HostTracker."
|
182
|
-
host_tracker
|
183
|
-
host_tracker.verbose=options[:verbose]
|
183
|
+
puts puts "Invoke the HostTracker with option: #{options[:target]}."
|
184
|
+
#host_tracker.verbose=options[:verbose]
|
184
185
|
else
|
185
186
|
abort "Error firing up HostTracker instance!"
|
186
187
|
end
|
data/lib/wmap/cidr_tracker.rb
CHANGED
@@ -18,8 +18,8 @@ class Wmap::CidrTracker
|
|
18
18
|
@verbose=params.fetch(:verbose, false)
|
19
19
|
@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
|
20
20
|
Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
|
21
|
-
@cidr_seeds=params.fetch(:cidr_seeds, @data_dir + 'cidrs')
|
22
|
-
File.
|
21
|
+
@cidr_seeds=params.fetch(:cidr_seeds, @data_dir + '/' + 'cidrs')
|
22
|
+
File.new(@cidr_seeds, "w") unless File.exist?(@cidr_seeds)
|
23
23
|
load_cidr_blks_from_file(@cidr_seeds)
|
24
24
|
end
|
25
25
|
|
@@ -48,7 +48,7 @@ class Wmap::CidrTracker
|
|
48
48
|
# 'setter' to load the known CIDR blocks into an instance variable @known_cidr_blks
|
49
49
|
def load_cidr_blks_from_file(file_cidrs=@cidr_seeds)
|
50
50
|
puts "Load the known CIDR seed file: #{file_cidrs}" if @verbose
|
51
|
-
f=File.open(file_cidrs, 'r')
|
51
|
+
f=File.open(file_cidrs, 'r', :encoding => 'UTF-8')
|
52
52
|
f.each do |line|
|
53
53
|
entry=line.chomp.split(',')
|
54
54
|
next unless is_cidr?(entry[0])
|
@@ -27,7 +27,7 @@ class SubDomain < Wmap::DomainTracker
|
|
27
27
|
@max_parallel=params.fetch(:max_parallel, 40)
|
28
28
|
# Hash table to hold the trusted domains
|
29
29
|
@sub_domains_file=params.fetch(:sub_domains_file, @data_dir + 'sub_domains')
|
30
|
-
File.
|
30
|
+
File.new(@sub_domains_file, "w") unless File.exist?(@sub_domains_file)
|
31
31
|
@known_internet_sub_domains=load_domains_from_file(@sub_domains_file) #unless @known_internet_sub_domains.size>0
|
32
32
|
end
|
33
33
|
|
@@ -70,7 +70,7 @@ class SubDomain < Wmap::DomainTracker
|
|
70
70
|
end
|
71
71
|
end
|
72
72
|
@known_internet_sub_domains.merge!(results)
|
73
|
-
puts "Done loading entries."
|
73
|
+
puts "Done loading sub_domain entries."
|
74
74
|
return results
|
75
75
|
else
|
76
76
|
puts "Error: no entry is loaded. Please check your list and try again."
|
data/lib/wmap/domain_tracker.rb
CHANGED
@@ -25,7 +25,7 @@ class Wmap::DomainTracker
|
|
25
25
|
@domains_file=params.fetch(:domains_file, @data_dir+'domains')
|
26
26
|
@max_parallel=params.fetch(:max_parallel, 40)
|
27
27
|
# Hash table to hold the trusted domains
|
28
|
-
File.
|
28
|
+
File.new(@domains_file, "w") unless File.exist?(@domains_file)
|
29
29
|
load_domains_from_file(@domains_file)
|
30
30
|
end
|
31
31
|
|
@@ -169,7 +169,7 @@ class Wmap::DomainTracker
|
|
169
169
|
end
|
170
170
|
end
|
171
171
|
@known_internet_domains.merge!(results)
|
172
|
-
puts "Done loading entries."
|
172
|
+
puts "Done loading domain entries."
|
173
173
|
return results
|
174
174
|
else
|
175
175
|
puts "Error: no entry is loaded. Please check your list and try again."
|
@@ -262,7 +262,7 @@ class Wmap::DomainTracker
|
|
262
262
|
when "Wmap::DomainTracker::SubDomain"
|
263
263
|
return @known_internet_sub_domains.key?(domain)
|
264
264
|
else
|
265
|
-
return
|
265
|
+
return false
|
266
266
|
end
|
267
267
|
rescue => ee
|
268
268
|
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
@@ -21,11 +21,11 @@ module Wmap
|
|
21
21
|
# Initialize the instance variables
|
22
22
|
def initialize (params = {})
|
23
23
|
@verbose=params.fetch(:verbose, false)
|
24
|
-
|
24
|
+
@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../../data/')
|
25
25
|
# Set default instance variables
|
26
26
|
@hosts_file=params.fetch(:hosts_file, @data_dir + 'prime_hosts')
|
27
27
|
# Initialize the instance variables
|
28
|
-
|
28
|
+
File.new(@hosts_file, "w") unless File.exist?(@hosts_file)
|
29
29
|
@known_hosts=load_known_hosts_from_file(@hosts_file)
|
30
30
|
@known_ips=Hash.new
|
31
31
|
de_duplicate
|
data/lib/wmap/host_tracker.rb
CHANGED
@@ -26,16 +26,16 @@ class Wmap::HostTracker
|
|
26
26
|
@hosts_file=params.fetch(:hosts_file, @data_dir + 'hosts')
|
27
27
|
@max_parallel=params.fetch(:max_parallel, 40)
|
28
28
|
# Initialize the instance variables
|
29
|
-
File.
|
30
|
-
load_known_hosts_from_file(@hosts_file)
|
29
|
+
File.new(@hosts_file, "w") unless File.exist?(@hosts_file)
|
30
|
+
@known_hosts=load_known_hosts_from_file(@hosts_file)
|
31
31
|
end
|
32
32
|
|
33
33
|
# Setter to load the known hosts from the local hosts file into a class instance
|
34
34
|
def load_known_hosts_from_file (f_hosts=@hosts_file)
|
35
35
|
puts "Loading local hosts from file: #{f_hosts} ..." if @verbose
|
36
|
-
|
36
|
+
known_hosts=Hash.new
|
37
37
|
@alias = Hash.new
|
38
|
-
File.
|
38
|
+
File.new(f_hosts, "w") unless File.exist?(f_hosts)
|
39
39
|
f=File.open(f_hosts, 'r')
|
40
40
|
f.each do |line|
|
41
41
|
next unless line =~ /\d+\.\d+\.\d+\.\d+/
|
@@ -43,11 +43,11 @@ class Wmap::HostTracker
|
|
43
43
|
key=entry[0].downcase
|
44
44
|
value=entry[1]
|
45
45
|
puts "Loading key value pair: #{key} - #{value}" if @verbose
|
46
|
-
|
47
|
-
|
46
|
+
known_hosts[key] = Hash.new unless known_hosts.key?(key)
|
47
|
+
known_hosts[key]= value
|
48
48
|
# For reverse host lookup
|
49
|
-
|
50
|
-
|
49
|
+
known_hosts[value] = Hash.new unless known_hosts.key?(value)
|
50
|
+
known_hosts[value] = key
|
51
51
|
# Count the number of alias for the recorded IP
|
52
52
|
if @alias.key?(value)
|
53
53
|
@alias[value]+=1
|
@@ -56,11 +56,12 @@ class Wmap::HostTracker
|
|
56
56
|
end
|
57
57
|
end
|
58
58
|
f.close
|
59
|
-
return @known_hosts
|
60
|
-
rescue => ee
|
61
|
-
puts "Exception on method #{__method__}: #{ee}"
|
62
59
|
return known_hosts
|
60
|
+
#rescue => ee
|
61
|
+
# puts "Exception on method #{__method__}: #{ee}"
|
62
|
+
# return known_hosts
|
63
63
|
end
|
64
|
+
alias_method :load, :load_known_hosts_from_file
|
64
65
|
|
65
66
|
# Save the current local hosts hash table into a (random) data repository file
|
66
67
|
def save_known_hosts_to_file!(f_hosts=@hosts_file)
|
@@ -96,30 +97,42 @@ class Wmap::HostTracker
|
|
96
97
|
puts "Exception on method #{__method__}: #{ee}"
|
97
98
|
end
|
98
99
|
|
100
|
+
# determine if host is part of trusted (known) root domains
|
101
|
+
def is_trusted?(host)
|
102
|
+
puts "Determin if host #{host} is part of trusted root domains" if @verbose
|
103
|
+
root=get_domain_root(host)
|
104
|
+
puts "Domain root: #{root}" if @verbose
|
105
|
+
domain_tracker=Wmap::DomainTracker.instance
|
106
|
+
domain_tracker.data_dir=@data_dir
|
107
|
+
domain_tracker.domains_file = domain_tracker.data_dir + "/" + "domains"
|
108
|
+
domain_tracker.load_domains_from_file
|
109
|
+
if domain_tracker.domain_known?(root)
|
110
|
+
domain_tracker=nil
|
111
|
+
return true
|
112
|
+
else
|
113
|
+
domain_tracker=nil
|
114
|
+
return false
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
99
118
|
# Setter to add host entry to the cache once at a time
|
100
119
|
def add(host)
|
101
120
|
puts "Add entry to the local host repository: #{host}"
|
102
121
|
host=host.strip.downcase unless host.nil?
|
122
|
+
root=get_domain_root(host)
|
103
123
|
unless @known_hosts.key?(host)
|
104
124
|
ip=host_2_ip(host)
|
105
125
|
record=Hash.new
|
106
126
|
if is_ip?(ip)
|
107
127
|
# filter host to known domains only
|
108
|
-
|
109
|
-
puts "Domain root: #{root}" if @verbose
|
110
|
-
domain_tracker=Wmap::DomainTracker.instance
|
111
|
-
domain_tracker.data_dir=@data_dir
|
112
|
-
domain_tracker.domains_file = domain_tracker.data_dir + "domains"
|
113
|
-
domain_tracker.load_domains_from_file
|
114
|
-
if domain_tracker.domain_known?(root)
|
115
|
-
domain_tracker=nil
|
128
|
+
if is_trusted?(host)
|
116
129
|
record[host]=ip
|
117
130
|
record[ip]=host
|
118
131
|
puts "Host data repository entry loaded: #{host} <=> #{ip}"
|
119
132
|
# Replace instance with the class variable to avoid potential race condition under parallel engine
|
120
133
|
# add additional logic to update the sub-domain table as well, 02/10/2014
|
121
134
|
sub=get_sub_domain(host)
|
122
|
-
if sub!=
|
135
|
+
if sub!=nil
|
123
136
|
tracker=Wmap::DomainTracker::SubDomain.instance
|
124
137
|
tracker.data_dir=@data_dir
|
125
138
|
tracker.sub_domains_file = tracker.data_dir + "sub_domains"
|
@@ -142,8 +155,8 @@ class Wmap::HostTracker
|
|
142
155
|
else
|
143
156
|
puts "Host is already exist. Skip: #{host}"
|
144
157
|
end
|
145
|
-
rescue => ee
|
146
|
-
|
158
|
+
#rescue => ee
|
159
|
+
# puts "Exception on method #{__method__}: #{ee}" if @verbose
|
147
160
|
end
|
148
161
|
|
149
162
|
# Setter to add host entry to the local hosts in batch (from an array)
|
@@ -164,7 +177,7 @@ class Wmap::HostTracker
|
|
164
177
|
end
|
165
178
|
end
|
166
179
|
@known_hosts.merge!(results)
|
167
|
-
puts "Done loading entries."
|
180
|
+
puts "Done loading host entries."
|
168
181
|
return results
|
169
182
|
else
|
170
183
|
puts "Error: empty list - no entry is loaded. Please check your input list and try again."
|
@@ -27,7 +27,7 @@ class DeactivatedSite < Wmap::SiteTracker
|
|
27
27
|
@sites_file=params.fetch(:sites_file, @data_dir + 'deactivated_sites')
|
28
28
|
@verbose=params.fetch(:verbose, false)
|
29
29
|
# Hash table to hold the site store
|
30
|
-
File.
|
30
|
+
File.new(@sites_file, "w") unless File.exist?(@sites_file)
|
31
31
|
@known_sites=load_site_stores_from_file(@sites_file)
|
32
32
|
end
|
33
33
|
|
@@ -25,13 +25,13 @@ class WpTracker < Wmap::SiteTracker
|
|
25
25
|
@verbose=params.fetch(:verbose, false)
|
26
26
|
@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../../data/')
|
27
27
|
Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
|
28
|
-
|
28
|
+
@sites_wp=params.fetch(:sites_wp, @data_dir+"wp_sites")
|
29
29
|
@http_timeout=params.fetch(:http_timeout, 5000)
|
30
30
|
@max_parallel=params.fetch(:max_parallel, 40)
|
31
31
|
Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
|
32
32
|
@log_file=@data_dir + "wp_checker.log"
|
33
|
-
File.
|
34
|
-
|
33
|
+
File.new(@sites_wp, "w") unless File.exist?(@sites_wp)
|
34
|
+
load_from_file(@sites_wp)
|
35
35
|
end
|
36
36
|
|
37
37
|
# 'setter' to load the known wordpress sites into an instance variable
|
@@ -174,7 +174,7 @@ class WpTracker < Wmap::SiteTracker
|
|
174
174
|
end
|
175
175
|
end
|
176
176
|
@known_wp_sites.merge!(results)
|
177
|
-
puts "Done loading entries."
|
177
|
+
puts "Done loading wp entries."
|
178
178
|
return results
|
179
179
|
else
|
180
180
|
puts "Error: no entry is loaded. Please check your list and try again."
|
data/lib/wmap/site_tracker.rb
CHANGED
@@ -25,7 +25,7 @@ class Wmap::SiteTracker
|
|
25
25
|
@sites_file=params.fetch(:sites_file, @data_dir+'sites')
|
26
26
|
@verbose=params.fetch(:verbose, false)
|
27
27
|
@max_parallel=params.fetch(:max_parallel, 30)
|
28
|
-
File.
|
28
|
+
File.new(@sites_file, "w") unless File.exist?(@sites_file)
|
29
29
|
# Hash table to hold the site store
|
30
30
|
load_site_stores_from_file(@sites_file)
|
31
31
|
end
|
@@ -34,7 +34,7 @@ class Wmap::SiteTracker
|
|
34
34
|
def load_site_stores_from_file (file=@sites_file)
|
35
35
|
puts "Loading the site store data repository from file: #{file} " if @verbose
|
36
36
|
@known_sites=Hash.new
|
37
|
-
File.
|
37
|
+
File.new(file, "w") unless File.exist?(file)
|
38
38
|
f=File.open(file, 'r')
|
39
39
|
f.each do |line|
|
40
40
|
line=line.chomp.strip
|
@@ -77,7 +77,9 @@ class Wmap::SiteTracker
|
|
77
77
|
f.write "# Local site store created by class #{self.class} method #{__method__} at: #{timestamp}\n"
|
78
78
|
f.write "# Website,Primary IP,Port,Hosting Status,Server,Response Code,MD5 Finger-print,Redirection,Timestamp\n"
|
79
79
|
@known_sites.keys.sort.map do |key|
|
80
|
-
|
80
|
+
if is_trusted?(key)
|
81
|
+
f.write "#{key},#{@known_sites[key]['ip']},#{@known_sites[key]['port']},#{@known_sites[key]['status']},#{@known_sites[key]['server']},#{@known_sites[key]['code']},#{@known_sites[key]['md5']},#{@known_sites[key]['redirection']},#{@known_sites[key]['timestamp']}\n"
|
82
|
+
end
|
81
83
|
end
|
82
84
|
f.close
|
83
85
|
puts "site store table is successfully saved: #{file_sites}"
|
@@ -94,6 +96,24 @@ class Wmap::SiteTracker
|
|
94
96
|
puts "Exception on method #{__method__}: #{ee}"
|
95
97
|
end
|
96
98
|
|
99
|
+
# determine site is trusted based on the known domains
|
100
|
+
def is_trusted?(site)
|
101
|
+
trusted=false
|
102
|
+
host=url_2_host(site)
|
103
|
+
root=get_domain_root(host)
|
104
|
+
domain_tracker=Wmap::DomainTracker.instance
|
105
|
+
domain_tracker.data_dir=@data_dir
|
106
|
+
domain_tracker.domains_file=@data_dir + "/" + "domains"
|
107
|
+
File.new(domain_tracker.domains_file, "w") unless File.exist?(domain_tracker.domains_file)
|
108
|
+
domain_tracker.load_domains_from_file(domain_tracker.domains_file)
|
109
|
+
trusted=domain_tracker.domain_known?(root)
|
110
|
+
domain_tracker=nil
|
111
|
+
return trusted
|
112
|
+
rescue => ee
|
113
|
+
puts "Exception on method #{__method__}: #{ee}"
|
114
|
+
return trusted
|
115
|
+
end
|
116
|
+
|
97
117
|
# Setter to add site entry to the cache one at a time
|
98
118
|
def add(site)
|
99
119
|
puts "Add entry to the site store: #{site}"
|
@@ -113,7 +133,7 @@ class Wmap::SiteTracker
|
|
113
133
|
# Additional logic to refresh deactivated site, 02/12/2014
|
114
134
|
deact=Wmap::SiteTracker::DeactivatedSite.instance
|
115
135
|
deact.sites_file=@data_dir + "/" + "deactivated_sites"
|
116
|
-
File.
|
136
|
+
File.new(deact.sites_file, "w") unless File.exist?(deact.sites_file)
|
117
137
|
deact.load_site_stores_from_file
|
118
138
|
# only trust either the domain or IP we know
|
119
139
|
if is_ip?(host)
|
@@ -125,13 +145,17 @@ class Wmap::SiteTracker
|
|
125
145
|
else
|
126
146
|
domain_tracker=Wmap::DomainTracker.instance
|
127
147
|
domain_tracker.domains_file=@data_dir + "/" + "domains"
|
128
|
-
File.
|
148
|
+
File.new(domain_tracker.domains_file, "w") unless File.exist?(domain_tracker.domains_file)
|
129
149
|
domain_tracker.load_domains_from_file(domain_tracker.domains_file)
|
130
150
|
trusted=domain_tracker.domain_known?(root)
|
131
151
|
domain_tracker=nil
|
132
152
|
end
|
133
153
|
end
|
134
154
|
# add record only if trusted
|
155
|
+
host_tracker = Wmap::HostTracker.instance
|
156
|
+
host_tracker.data_dir= @data_dir
|
157
|
+
host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
|
158
|
+
host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
|
135
159
|
if trusted
|
136
160
|
# Add logic to check site status before adding it
|
137
161
|
checker=Wmap::UrlChecker.new(:data_dir=>@data_dir).check(site)
|
@@ -144,10 +168,6 @@ class Wmap::SiteTracker
|
|
144
168
|
raise "Site is currently down. Skip #{site}" if checker['code']==10000
|
145
169
|
end
|
146
170
|
raise "Exception on add method - Fail to resolve the host-name: Host - #{host}, IP - #{ip}. Skip #{site}" unless is_ip?(ip)
|
147
|
-
host_tracker = Wmap::HostTracker.instance
|
148
|
-
host_tracker.data_dir= @data_dir
|
149
|
-
host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
|
150
|
-
host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
|
151
171
|
# Update the local host table when necessary
|
152
172
|
if is_ip?(host)
|
153
173
|
# Case #1: Trusted site contains IP
|
@@ -291,7 +311,7 @@ class Wmap::SiteTracker
|
|
291
311
|
# Additional logic to deactivate the site properly, by moving it to the DeactivatedSite list, 02/07/2014
|
292
312
|
deact=Wmap::SiteTracker::DeactivatedSite.instance
|
293
313
|
deact.sites_file=@data_dir + 'deactivated_sites'
|
294
|
-
File.
|
314
|
+
File.new(deact.sites_file, "w") unless File.exist?(deact.sites_file)
|
295
315
|
site=site.strip.downcase
|
296
316
|
site=url_2_site(site)
|
297
317
|
if @known_sites.key?(site)
|
@@ -341,8 +361,8 @@ class Wmap::SiteTracker
|
|
341
361
|
else
|
342
362
|
puts "Error: no entry is loaded. Please check your list and try again."
|
343
363
|
end
|
344
|
-
|
345
|
-
|
364
|
+
rescue => ee
|
365
|
+
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
346
366
|
end
|
347
367
|
alias_method :dels, :bulk_delete
|
348
368
|
|
data/lib/wmap/url_checker.rb
CHANGED
@@ -88,9 +88,9 @@ class Wmap::UrlChecker
|
|
88
88
|
checker['redirection']=nil
|
89
89
|
checker['timestamp']=timestamp
|
90
90
|
return checker
|
91
|
-
rescue Exception => ee
|
92
|
-
|
93
|
-
|
91
|
+
#rescue Exception => ee
|
92
|
+
# puts "Exception on method #{__method__} for #{url}: #{ee}" # if @verbose
|
93
|
+
# return nil
|
94
94
|
end
|
95
95
|
alias_method :check, :url_worker
|
96
96
|
|
@@ -28,7 +28,7 @@ module Wmap
|
|
28
28
|
file=params.fetch(:signature_file, @signature_file)
|
29
29
|
@tag_signatures=load_sig_from_file(file)
|
30
30
|
@tag_file=params.fetch(:tag_file, @data_dir + 'tag_sites')
|
31
|
-
File.
|
31
|
+
File.new(@tag_file, "w") unless File.exist?(@tag_file)
|
32
32
|
# load the known tag store
|
33
33
|
load_tag_from_file(@tag_file)
|
34
34
|
@landings = Hash.new # cache landing page to reduce redundant browsing
|
@@ -119,7 +119,7 @@ module Wmap
|
|
119
119
|
end
|
120
120
|
end
|
121
121
|
@tag_store.merge!(results)
|
122
|
-
puts "Done loading entries."
|
122
|
+
puts "Done loading adware entries."
|
123
123
|
tags = nil
|
124
124
|
return results
|
125
125
|
else
|
data/lib/wmap/url_crawler.rb
CHANGED
@@ -90,7 +90,7 @@ class Wmap::UrlCrawler
|
|
90
90
|
|
91
91
|
# The worker instance of crawler who perform the labour work
|
92
92
|
def crawl_worker(url0)
|
93
|
-
puts "Please be aware that it may take a while to crawl #{url0}, depending on the site's responsiveness and
|
93
|
+
puts "Please be aware that it may take a while to crawl #{url0}, depending on the site's responsiveness and discovery contents."
|
94
94
|
# Input URL sanity check first
|
95
95
|
if is_url?(url0)
|
96
96
|
host=url_2_host(url0)
|
@@ -1,20 +1,15 @@
|
|
1
1
|
# Default list of Google search keywords (known company legal names, for example) for the service discovery, one entry at per line
|
2
2
|
# Used by the Wmap::GoogleSearchScraper class
|
3
|
+
Google llc
|
3
4
|
|
4
5
|
# Official whole name
|
5
|
-
Penguin Random House
|
6
6
|
|
7
7
|
# 2014 purchase
|
8
|
-
Random House
|
9
8
|
|
10
9
|
# Parent company
|
11
|
-
Bertelsmann
|
12
10
|
|
13
11
|
# Division
|
14
|
-
Crown Books
|
15
12
|
|
16
13
|
# Division
|
17
|
-
Dorling Kindersley
|
18
14
|
|
19
15
|
# Division
|
20
|
-
Penguin Random House Canada
|
data/version.txt
CHANGED
@@ -3,8 +3,8 @@
|
|
3
3
|
###############################################################################
|
4
4
|
package = wmap
|
5
5
|
# wmap version 2.0 == web_discovery version 1.5.3
|
6
|
-
version = 2.8.
|
7
|
-
date =
|
6
|
+
version = 2.8.4
|
7
|
+
date = 2024-07-01
|
8
8
|
|
9
9
|
author = Sam (Yang) Li
|
10
10
|
email = yang.li@owasp.org
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: wmap
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.8.
|
4
|
+
version: 2.8.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sam (Yang) Li
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-07-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: dnsruby
|