wmap 2.8.2 → 2.8.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 2dee2577809daef8231ddc3d97a7e0abab5312700f36ab2233366ff6729ae388
4
- data.tar.gz: 5dee805b28f88e2ae320e6afd9b4ef7d6f25bff99a34cf46b42c32249bf7eaad
3
+ metadata.gz: 6ebcb2cd5065f39431d72629e23c6843927c586c474f0e4e94d4dbb222848cf8
4
+ data.tar.gz: 0d5b4655157d24e80e015e02e4c3ca6783f41e4bf8385e7b89fb4d0078b96356
5
5
  SHA512:
6
- metadata.gz: c78742dad1d356b88d2b45a0773527fd8f4e54cbff13c0ccb5f9fd9a228fac56cc2c26a705e8c5ce068b4d2bdf50d1fd190060501a1fc17e0a1040e583080db6
7
- data.tar.gz: 7eea2eaa6ee45e9ac7857eaa603177b00424a6842cdd89a200062bdcd737c1a66fa2310645e80272fce71d0193fb9ca8044f232e0dabdf20c2da453e436c84cc
6
+ metadata.gz: b9b786babd3b056fa637867b00fdd71444ca4e48e73df640d5bfd56f3ed0ce4170babd73f9ea3f0f6e6f310c5f245d80b8857b2d9e2ba1180c0cdb4e1b1cab3a
7
+ data.tar.gz: b2d875d0a8428e6de815be4d051c94d14412b9643a2eb018e8db8d1024c311b42251a95a0a7818659582478d3a08742a27e1400ac71a4e825f982cb2ed95e455
data/README.md CHANGED
@@ -21,7 +21,7 @@ Note that program is mainly operating on Command Line Interface (CLI). For bette
21
21
 
22
22
 
23
23
  ## WMAP in Motion
24
- You can try out the complete [demo web app](http://wmap.io/) deployed in the DigitalOcean cloud: http://wmap.io/
24
+ You can try out the complete [demo web app](https://www.wmap.cloud/) deployed in the DigitalOcean cloud: https://www.wmap.cloud/
25
25
 
26
26
 
27
27
  ## Installation
data/bin/wmap CHANGED
@@ -146,41 +146,42 @@ when nil,[]
146
146
  puts "No new site found. There is no change to the site tracking data repository. "
147
147
  else
148
148
  puts "Automatically save the discovery results into the site tracking data repository: "
149
+ inventory = Wmap::SiteTracker.instance
149
150
  if options[:target] && options[:data_dir]
150
151
  puts "Start the SiteTracker with the optional directory setter. "
151
- inventory=Wmap::SiteTracker.instance
152
152
  inventory.data_dir = options[:data_dir]
153
+ #inventory.verbose = true
153
154
  inventory.sites_file = inventory.data_dir + "/" + "sites"
154
155
  inventory.load_site_stores_from_file(inventory.sites_file)
155
156
  elsif options[:target]
156
157
  puts "Start the SiteTracker. "
157
- inventory=Wmap::SiteTracker.instance
158
158
  else
159
159
  abort "Error firing up SiteTracker instance!"
160
160
  end
161
161
  new_sites=inventory.adds(dis_sites.keys-["",nil])
162
- if new_sites.size>0 && options[:data_dir]
162
+ puts "Newly discovery sties: #{new_sites}"
163
+ if options[:data_dir]
163
164
  inventory.save!(inventory.sites_file)
164
- elsif new_sites.size>0
165
+ else
165
166
  inventory.save!
166
167
  end
167
168
  inventory=nil
168
- puts "Done! New found sites are successfully saved. " if new_sites.size > 0
169
+ puts "Done! New found sites are successfully saved. " unless new_sites.nil? or new_sites.empty?
169
170
  end
170
171
 
171
172
 
172
173
  # seventh step - update the hosts repository
174
+ puts "Invoke the HostTracker with optional directory setter."
175
+ host_tracker = Wmap::HostTracker.instance
173
176
  if options[:target] && options[:data_dir]
174
- puts "Invoke the HostTracker with optional directory setter."
175
- host_tracker = Wmap::HostTracker.instance
177
+ puts puts "Invoke the HostTracker with options: #{options[:data_dir]}, #{options[:target]}"
176
178
  host_tracker.verbose=options[:verbose]
177
179
  host_tracker.data_dir = options[:data_dir]
178
180
  host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
179
181
  host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
180
182
  elsif options[:target]
181
- puts puts "Invoke the HostTracker."
182
- host_tracker = Wmap::HostTracker.instance
183
- host_tracker.verbose=options[:verbose]
183
+ puts puts "Invoke the HostTracker with option: #{options[:target]}."
184
+ #host_tracker.verbose=options[:verbose]
184
185
  else
185
186
  abort "Error firing up HostTracker instance!"
186
187
  end
@@ -18,8 +18,8 @@ class Wmap::CidrTracker
18
18
  @verbose=params.fetch(:verbose, false)
19
19
  @data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
20
20
  Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
21
- @cidr_seeds=params.fetch(:cidr_seeds, @data_dir + 'cidrs')
22
- File.write(@cidr_seeds, "") unless File.exist?(@cidr_seeds)
21
+ @cidr_seeds=params.fetch(:cidr_seeds, @data_dir + '/' + 'cidrs')
22
+ File.new(@cidr_seeds, "w") unless File.exist?(@cidr_seeds)
23
23
  load_cidr_blks_from_file(@cidr_seeds)
24
24
  end
25
25
 
@@ -48,7 +48,7 @@ class Wmap::CidrTracker
48
48
  # 'setter' to load the known CIDR blocks into an instance variable @known_cidr_blks
49
49
  def load_cidr_blks_from_file(file_cidrs=@cidr_seeds)
50
50
  puts "Load the known CIDR seed file: #{file_cidrs}" if @verbose
51
- f=File.open(file_cidrs, 'r')
51
+ f=File.open(file_cidrs, 'r', :encoding => 'UTF-8')
52
52
  f.each do |line|
53
53
  entry=line.chomp.split(',')
54
54
  next unless is_cidr?(entry[0])
@@ -27,7 +27,7 @@ class SubDomain < Wmap::DomainTracker
27
27
  @max_parallel=params.fetch(:max_parallel, 40)
28
28
  # Hash table to hold the trusted domains
29
29
  @sub_domains_file=params.fetch(:sub_domains_file, @data_dir + 'sub_domains')
30
- File.write(@sub_domains_file, "") unless File.exist?(@sub_domains_file)
30
+ File.new(@sub_domains_file, "w") unless File.exist?(@sub_domains_file)
31
31
  @known_internet_sub_domains=load_domains_from_file(@sub_domains_file) #unless @known_internet_sub_domains.size>0
32
32
  end
33
33
 
@@ -70,7 +70,7 @@ class SubDomain < Wmap::DomainTracker
70
70
  end
71
71
  end
72
72
  @known_internet_sub_domains.merge!(results)
73
- puts "Done loading entries."
73
+ puts "Done loading sub_domain entries."
74
74
  return results
75
75
  else
76
76
  puts "Error: no entry is loaded. Please check your list and try again."
@@ -25,7 +25,7 @@ class Wmap::DomainTracker
25
25
  @domains_file=params.fetch(:domains_file, @data_dir+'domains')
26
26
  @max_parallel=params.fetch(:max_parallel, 40)
27
27
  # Hash table to hold the trusted domains
28
- File.write(@domains_file, "") unless File.exist?(@domains_file)
28
+ File.new(@domains_file, "w") unless File.exist?(@domains_file)
29
29
  load_domains_from_file(@domains_file)
30
30
  end
31
31
 
@@ -169,7 +169,7 @@ class Wmap::DomainTracker
169
169
  end
170
170
  end
171
171
  @known_internet_domains.merge!(results)
172
- puts "Done loading entries."
172
+ puts "Done loading domain entries."
173
173
  return results
174
174
  else
175
175
  puts "Error: no entry is loaded. Please check your list and try again."
@@ -262,7 +262,7 @@ class Wmap::DomainTracker
262
262
  when "Wmap::DomainTracker::SubDomain"
263
263
  return @known_internet_sub_domains.key?(domain)
264
264
  else
265
- return nil
265
+ return false
266
266
  end
267
267
  rescue => ee
268
268
  puts "Exception on method #{__method__}: #{ee}" if @verbose
@@ -21,11 +21,11 @@ module Wmap
21
21
  # Initialize the instance variables
22
22
  def initialize (params = {})
23
23
  @verbose=params.fetch(:verbose, false)
24
- @data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../../data/')
24
+ @data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../../data/')
25
25
  # Set default instance variables
26
26
  @hosts_file=params.fetch(:hosts_file, @data_dir + 'prime_hosts')
27
27
  # Initialize the instance variables
28
- File.write(@hosts_file, "") unless File.exist?(@hosts_file)
28
+ File.new(@hosts_file, "w") unless File.exist?(@hosts_file)
29
29
  @known_hosts=load_known_hosts_from_file(@hosts_file)
30
30
  @known_ips=Hash.new
31
31
  de_duplicate
@@ -26,16 +26,16 @@ class Wmap::HostTracker
26
26
  @hosts_file=params.fetch(:hosts_file, @data_dir + 'hosts')
27
27
  @max_parallel=params.fetch(:max_parallel, 40)
28
28
  # Initialize the instance variables
29
- File.write(@hosts_file, "") unless File.exist?(@hosts_file)
30
- load_known_hosts_from_file(@hosts_file)
29
+ File.new(@hosts_file, "w") unless File.exist?(@hosts_file)
30
+ @known_hosts=load_known_hosts_from_file(@hosts_file)
31
31
  end
32
32
 
33
33
  # Setter to load the known hosts from the local hosts file into a class instance
34
34
  def load_known_hosts_from_file (f_hosts=@hosts_file)
35
35
  puts "Loading local hosts from file: #{f_hosts} ..." if @verbose
36
- @known_hosts=Hash.new
36
+ known_hosts=Hash.new
37
37
  @alias = Hash.new
38
- File.write(f_hosts, "") unless File.exist?(f_hosts)
38
+ File.new(f_hosts, "w") unless File.exist?(f_hosts)
39
39
  f=File.open(f_hosts, 'r')
40
40
  f.each do |line|
41
41
  next unless line =~ /\d+\.\d+\.\d+\.\d+/
@@ -43,11 +43,11 @@ class Wmap::HostTracker
43
43
  key=entry[0].downcase
44
44
  value=entry[1]
45
45
  puts "Loading key value pair: #{key} - #{value}" if @verbose
46
- @known_hosts[key] = Hash.new unless @known_hosts.key?(key)
47
- @known_hosts[key]= value
46
+ known_hosts[key] = Hash.new unless known_hosts.key?(key)
47
+ known_hosts[key]= value
48
48
  # For reverse host lookup
49
- @known_hosts[value] = Hash.new unless @known_hosts.key?(value)
50
- @known_hosts[value] = key
49
+ known_hosts[value] = Hash.new unless known_hosts.key?(value)
50
+ known_hosts[value] = key
51
51
  # Count the number of alias for the recorded IP
52
52
  if @alias.key?(value)
53
53
  @alias[value]+=1
@@ -56,11 +56,12 @@ class Wmap::HostTracker
56
56
  end
57
57
  end
58
58
  f.close
59
- return @known_hosts
60
- rescue => ee
61
- puts "Exception on method #{__method__}: #{ee}"
62
59
  return known_hosts
60
+ #rescue => ee
61
+ # puts "Exception on method #{__method__}: #{ee}"
62
+ # return known_hosts
63
63
  end
64
+ alias_method :load, :load_known_hosts_from_file
64
65
 
65
66
  # Save the current local hosts hash table into a (random) data repository file
66
67
  def save_known_hosts_to_file!(f_hosts=@hosts_file)
@@ -96,30 +97,42 @@ class Wmap::HostTracker
96
97
  puts "Exception on method #{__method__}: #{ee}"
97
98
  end
98
99
 
100
+ # determine if host is part of trusted (known) root domains
101
+ def is_trusted?(host)
102
+ puts "Determin if host #{host} is part of trusted root domains" if @verbose
103
+ root=get_domain_root(host)
104
+ puts "Domain root: #{root}" if @verbose
105
+ domain_tracker=Wmap::DomainTracker.instance
106
+ domain_tracker.data_dir=@data_dir
107
+ domain_tracker.domains_file = domain_tracker.data_dir + "/" + "domains"
108
+ domain_tracker.load_domains_from_file
109
+ if domain_tracker.domain_known?(root)
110
+ domain_tracker=nil
111
+ return true
112
+ else
113
+ domain_tracker=nil
114
+ return false
115
+ end
116
+ end
117
+
99
118
  # Setter to add host entry to the cache once at a time
100
119
  def add(host)
101
120
  puts "Add entry to the local host repository: #{host}"
102
121
  host=host.strip.downcase unless host.nil?
122
+ root=get_domain_root(host)
103
123
  unless @known_hosts.key?(host)
104
124
  ip=host_2_ip(host)
105
125
  record=Hash.new
106
126
  if is_ip?(ip)
107
127
  # filter host to known domains only
108
- root=get_domain_root(host)
109
- puts "Domain root: #{root}" if @verbose
110
- domain_tracker=Wmap::DomainTracker.instance
111
- domain_tracker.data_dir=@data_dir
112
- domain_tracker.domains_file = domain_tracker.data_dir + "domains"
113
- domain_tracker.load_domains_from_file
114
- if domain_tracker.domain_known?(root)
115
- domain_tracker=nil
128
+ if is_trusted?(host)
116
129
  record[host]=ip
117
130
  record[ip]=host
118
131
  puts "Host data repository entry loaded: #{host} <=> #{ip}"
119
132
  # Replace instance with the class variable to avoid potential race condition under parallel engine
120
133
  # add additional logic to update the sub-domain table as well, 02/10/2014
121
134
  sub=get_sub_domain(host)
122
- if sub!=root
135
+ if sub!=nil
123
136
  tracker=Wmap::DomainTracker::SubDomain.instance
124
137
  tracker.data_dir=@data_dir
125
138
  tracker.sub_domains_file = tracker.data_dir + "sub_domains"
@@ -142,8 +155,8 @@ class Wmap::HostTracker
142
155
  else
143
156
  puts "Host is already exist. Skip: #{host}"
144
157
  end
145
- rescue => ee
146
- puts "Exception on method #{__method__}: #{ee}" if @verbose
158
+ #rescue => ee
159
+ # puts "Exception on method #{__method__}: #{ee}" if @verbose
147
160
  end
148
161
 
149
162
  # Setter to add host entry to the local hosts in batch (from an array)
@@ -164,7 +177,7 @@ class Wmap::HostTracker
164
177
  end
165
178
  end
166
179
  @known_hosts.merge!(results)
167
- puts "Done loading entries."
180
+ puts "Done loading host entries."
168
181
  return results
169
182
  else
170
183
  puts "Error: empty list - no entry is loaded. Please check your input list and try again."
@@ -27,7 +27,7 @@ class DeactivatedSite < Wmap::SiteTracker
27
27
  @sites_file=params.fetch(:sites_file, @data_dir + 'deactivated_sites')
28
28
  @verbose=params.fetch(:verbose, false)
29
29
  # Hash table to hold the site store
30
- File.write(@sites_file, "") unless File.exist?(@sites_file)
30
+ File.new(@sites_file, "w") unless File.exist?(@sites_file)
31
31
  @known_sites=load_site_stores_from_file(@sites_file)
32
32
  end
33
33
 
@@ -25,13 +25,13 @@ class WpTracker < Wmap::SiteTracker
25
25
  @verbose=params.fetch(:verbose, false)
26
26
  @data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../../data/')
27
27
  Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
28
- @sites_wp=params.fetch(:sites_wp, @data_dir+"wp_sites")
28
+ @sites_wp=params.fetch(:sites_wp, @data_dir+"wp_sites")
29
29
  @http_timeout=params.fetch(:http_timeout, 5000)
30
30
  @max_parallel=params.fetch(:max_parallel, 40)
31
31
  Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
32
32
  @log_file=@data_dir + "wp_checker.log"
33
- File.write(@sites_wp, "") unless File.exist?(@sites_wp)
34
- load_from_file(@sites_wp)
33
+ File.new(@sites_wp, "w") unless File.exist?(@sites_wp)
34
+ load_from_file(@sites_wp)
35
35
  end
36
36
 
37
37
  # 'setter' to load the known wordpress sites into an instance variable
@@ -174,7 +174,7 @@ class WpTracker < Wmap::SiteTracker
174
174
  end
175
175
  end
176
176
  @known_wp_sites.merge!(results)
177
- puts "Done loading entries."
177
+ puts "Done loading wp entries."
178
178
  return results
179
179
  else
180
180
  puts "Error: no entry is loaded. Please check your list and try again."
@@ -25,7 +25,7 @@ class Wmap::SiteTracker
25
25
  @sites_file=params.fetch(:sites_file, @data_dir+'sites')
26
26
  @verbose=params.fetch(:verbose, false)
27
27
  @max_parallel=params.fetch(:max_parallel, 30)
28
- File.write(@sites_file, "") unless File.exist?(@sites_file)
28
+ File.new(@sites_file, "w") unless File.exist?(@sites_file)
29
29
  # Hash table to hold the site store
30
30
  load_site_stores_from_file(@sites_file)
31
31
  end
@@ -34,7 +34,7 @@ class Wmap::SiteTracker
34
34
  def load_site_stores_from_file (file=@sites_file)
35
35
  puts "Loading the site store data repository from file: #{file} " if @verbose
36
36
  @known_sites=Hash.new
37
- File.write(file, "") unless File.exist?(file)
37
+ File.new(file, "w") unless File.exist?(file)
38
38
  f=File.open(file, 'r')
39
39
  f.each do |line|
40
40
  line=line.chomp.strip
@@ -77,7 +77,9 @@ class Wmap::SiteTracker
77
77
  f.write "# Local site store created by class #{self.class} method #{__method__} at: #{timestamp}\n"
78
78
  f.write "# Website,Primary IP,Port,Hosting Status,Server,Response Code,MD5 Finger-print,Redirection,Timestamp\n"
79
79
  @known_sites.keys.sort.map do |key|
80
- f.write "#{key},#{@known_sites[key]['ip']},#{@known_sites[key]['port']},#{@known_sites[key]['status']},#{@known_sites[key]['server']},#{@known_sites[key]['code']},#{@known_sites[key]['md5']},#{@known_sites[key]['redirection']},#{@known_sites[key]['timestamp']}\n"
80
+ if is_trusted?(key)
81
+ f.write "#{key},#{@known_sites[key]['ip']},#{@known_sites[key]['port']},#{@known_sites[key]['status']},#{@known_sites[key]['server']},#{@known_sites[key]['code']},#{@known_sites[key]['md5']},#{@known_sites[key]['redirection']},#{@known_sites[key]['timestamp']}\n"
82
+ end
81
83
  end
82
84
  f.close
83
85
  puts "site store table is successfully saved: #{file_sites}"
@@ -94,6 +96,24 @@ class Wmap::SiteTracker
94
96
  puts "Exception on method #{__method__}: #{ee}"
95
97
  end
96
98
 
99
+ # determine site is trusted based on the known domains
100
+ def is_trusted?(site)
101
+ trusted=false
102
+ host=url_2_host(site)
103
+ root=get_domain_root(host)
104
+ domain_tracker=Wmap::DomainTracker.instance
105
+ domain_tracker.data_dir=@data_dir
106
+ domain_tracker.domains_file=@data_dir + "/" + "domains"
107
+ File.new(domain_tracker.domains_file, "w") unless File.exist?(domain_tracker.domains_file)
108
+ domain_tracker.load_domains_from_file(domain_tracker.domains_file)
109
+ trusted=domain_tracker.domain_known?(root)
110
+ domain_tracker=nil
111
+ return trusted
112
+ rescue => ee
113
+ puts "Exception on method #{__method__}: #{ee}"
114
+ return trusted
115
+ end
116
+
97
117
  # Setter to add site entry to the cache one at a time
98
118
  def add(site)
99
119
  puts "Add entry to the site store: #{site}"
@@ -113,7 +133,7 @@ class Wmap::SiteTracker
113
133
  # Additional logic to refresh deactivated site, 02/12/2014
114
134
  deact=Wmap::SiteTracker::DeactivatedSite.instance
115
135
  deact.sites_file=@data_dir + "/" + "deactivated_sites"
116
- File.write(deact.sites_file, "") unless File.exist?(deact.sites_file)
136
+ File.new(deact.sites_file, "w") unless File.exist?(deact.sites_file)
117
137
  deact.load_site_stores_from_file
118
138
  # only trust either the domain or IP we know
119
139
  if is_ip?(host)
@@ -125,13 +145,17 @@ class Wmap::SiteTracker
125
145
  else
126
146
  domain_tracker=Wmap::DomainTracker.instance
127
147
  domain_tracker.domains_file=@data_dir + "/" + "domains"
128
- File.write(domain_tracker.domains_file, "") unless File.exist?(domain_tracker.domains_file)
148
+ File.new(domain_tracker.domains_file, "w") unless File.exist?(domain_tracker.domains_file)
129
149
  domain_tracker.load_domains_from_file(domain_tracker.domains_file)
130
150
  trusted=domain_tracker.domain_known?(root)
131
151
  domain_tracker=nil
132
152
  end
133
153
  end
134
154
  # add record only if trusted
155
+ host_tracker = Wmap::HostTracker.instance
156
+ host_tracker.data_dir= @data_dir
157
+ host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
158
+ host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
135
159
  if trusted
136
160
  # Add logic to check site status before adding it
137
161
  checker=Wmap::UrlChecker.new(:data_dir=>@data_dir).check(site)
@@ -144,10 +168,6 @@ class Wmap::SiteTracker
144
168
  raise "Site is currently down. Skip #{site}" if checker['code']==10000
145
169
  end
146
170
  raise "Exception on add method - Fail to resolve the host-name: Host - #{host}, IP - #{ip}. Skip #{site}" unless is_ip?(ip)
147
- host_tracker = Wmap::HostTracker.instance
148
- host_tracker.data_dir= @data_dir
149
- host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
150
- host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
151
171
  # Update the local host table when necessary
152
172
  if is_ip?(host)
153
173
  # Case #1: Trusted site contains IP
@@ -291,7 +311,7 @@ class Wmap::SiteTracker
291
311
  # Additional logic to deactivate the site properly, by moving it to the DeactivatedSite list, 02/07/2014
292
312
  deact=Wmap::SiteTracker::DeactivatedSite.instance
293
313
  deact.sites_file=@data_dir + 'deactivated_sites'
294
- File.write(deact.sites_file, "") unless File.exist?(deact.sites_file)
314
+ File.new(deact.sites_file, "w") unless File.exist?(deact.sites_file)
295
315
  site=site.strip.downcase
296
316
  site=url_2_site(site)
297
317
  if @known_sites.key?(site)
@@ -341,8 +361,8 @@ class Wmap::SiteTracker
341
361
  else
342
362
  puts "Error: no entry is loaded. Please check your list and try again."
343
363
  end
344
- #rescue => ee
345
- # puts "Exception on method #{__method__}: #{ee}" if @verbose
364
+ rescue => ee
365
+ puts "Exception on method #{__method__}: #{ee}" if @verbose
346
366
  end
347
367
  alias_method :dels, :bulk_delete
348
368
 
@@ -88,9 +88,9 @@ class Wmap::UrlChecker
88
88
  checker['redirection']=nil
89
89
  checker['timestamp']=timestamp
90
90
  return checker
91
- rescue Exception => ee
92
- puts "Exception on method #{__method__} for #{url}: #{ee}" # if @verbose
93
- return nil
91
+ #rescue Exception => ee
92
+ # puts "Exception on method #{__method__} for #{url}: #{ee}" # if @verbose
93
+ # return nil
94
94
  end
95
95
  alias_method :check, :url_worker
96
96
 
@@ -28,7 +28,7 @@ module Wmap
28
28
  file=params.fetch(:signature_file, @signature_file)
29
29
  @tag_signatures=load_sig_from_file(file)
30
30
  @tag_file=params.fetch(:tag_file, @data_dir + 'tag_sites')
31
- File.write(@tag_file, "") unless File.exist?(@tag_file)
31
+ File.new(@tag_file, "w") unless File.exist?(@tag_file)
32
32
  # load the known tag store
33
33
  load_tag_from_file(@tag_file)
34
34
  @landings = Hash.new # cache landing page to reduce redundant browsing
@@ -119,7 +119,7 @@ module Wmap
119
119
  end
120
120
  end
121
121
  @tag_store.merge!(results)
122
- puts "Done loading entries."
122
+ puts "Done loading adware entries."
123
123
  tags = nil
124
124
  return results
125
125
  else
@@ -90,7 +90,7 @@ class Wmap::UrlCrawler
90
90
 
91
91
  # The worker instance of crawler who perform the labour work
92
92
  def crawl_worker(url0)
93
- puts "Please be aware that it may take a while to crawl #{url0}, depending on the site's responsiveness and the amount of contents."
93
+ puts "Please be aware that it may take a while to crawl #{url0}, depending on the site's responsiveness and discovery contents."
94
94
  # Input URL sanity check first
95
95
  if is_url?(url0)
96
96
  host=url_2_host(url0)
@@ -1,20 +1,15 @@
1
1
  # Default list of Google search keywords (known company legal names, for example) for the service discovery, one entry at per line
2
2
  # Used by the Wmap::GoogleSearchScraper class
3
+ Google llc
3
4
 
4
5
  # Official whole name
5
- Penguin Random House
6
6
 
7
7
  # 2014 purchase
8
- Random House
9
8
 
10
9
  # Parent company
11
- Bertelsmann
12
10
 
13
11
  # Division
14
- Crown Books
15
12
 
16
13
  # Division
17
- Dorling Kindersley
18
14
 
19
15
  # Division
20
- Penguin Random House Canada
data/version.txt CHANGED
@@ -3,8 +3,8 @@
3
3
  ###############################################################################
4
4
  package = wmap
5
5
  # wmap version 2.0 == web_discovery version 1.5.3
6
- version = 2.8.2
7
- date = 2021-07-23
6
+ version = 2.8.4
7
+ date = 2024-07-01
8
8
 
9
9
  author = Sam (Yang) Li
10
10
  email = yang.li@owasp.org
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: wmap
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.8.2
4
+ version: 2.8.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sam (Yang) Li
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-07-23 00:00:00.000000000 Z
11
+ date: 2024-07-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: dnsruby