wmap 2.8.2 → 2.8.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 2dee2577809daef8231ddc3d97a7e0abab5312700f36ab2233366ff6729ae388
4
- data.tar.gz: 5dee805b28f88e2ae320e6afd9b4ef7d6f25bff99a34cf46b42c32249bf7eaad
3
+ metadata.gz: 6ebcb2cd5065f39431d72629e23c6843927c586c474f0e4e94d4dbb222848cf8
4
+ data.tar.gz: 0d5b4655157d24e80e015e02e4c3ca6783f41e4bf8385e7b89fb4d0078b96356
5
5
  SHA512:
6
- metadata.gz: c78742dad1d356b88d2b45a0773527fd8f4e54cbff13c0ccb5f9fd9a228fac56cc2c26a705e8c5ce068b4d2bdf50d1fd190060501a1fc17e0a1040e583080db6
7
- data.tar.gz: 7eea2eaa6ee45e9ac7857eaa603177b00424a6842cdd89a200062bdcd737c1a66fa2310645e80272fce71d0193fb9ca8044f232e0dabdf20c2da453e436c84cc
6
+ metadata.gz: b9b786babd3b056fa637867b00fdd71444ca4e48e73df640d5bfd56f3ed0ce4170babd73f9ea3f0f6e6f310c5f245d80b8857b2d9e2ba1180c0cdb4e1b1cab3a
7
+ data.tar.gz: b2d875d0a8428e6de815be4d051c94d14412b9643a2eb018e8db8d1024c311b42251a95a0a7818659582478d3a08742a27e1400ac71a4e825f982cb2ed95e455
data/README.md CHANGED
@@ -21,7 +21,7 @@ Note that program is mainly operating on Command Line Interface (CLI). For bette
21
21
 
22
22
 
23
23
  ## WMAP in Motion
24
- You can try out the complete [demo web app](http://wmap.io/) deployed in the DigitalOcean cloud: http://wmap.io/
24
+ You can try out the complete [demo web app](https://www.wmap.cloud/) deployed in the DigitalOcean cloud: https://www.wmap.cloud/
25
25
 
26
26
 
27
27
  ## Installation
data/bin/wmap CHANGED
@@ -146,41 +146,42 @@ when nil,[]
146
146
  puts "No new site found. There is no change to the site tracking data repository. "
147
147
  else
148
148
  puts "Automatically save the discovery results into the site tracking data repository: "
149
+ inventory = Wmap::SiteTracker.instance
149
150
  if options[:target] && options[:data_dir]
150
151
  puts "Start the SiteTracker with the optional directory setter. "
151
- inventory=Wmap::SiteTracker.instance
152
152
  inventory.data_dir = options[:data_dir]
153
+ #inventory.verbose = true
153
154
  inventory.sites_file = inventory.data_dir + "/" + "sites"
154
155
  inventory.load_site_stores_from_file(inventory.sites_file)
155
156
  elsif options[:target]
156
157
  puts "Start the SiteTracker. "
157
- inventory=Wmap::SiteTracker.instance
158
158
  else
159
159
  abort "Error firing up SiteTracker instance!"
160
160
  end
161
161
  new_sites=inventory.adds(dis_sites.keys-["",nil])
162
- if new_sites.size>0 && options[:data_dir]
162
+ puts "Newly discovery sties: #{new_sites}"
163
+ if options[:data_dir]
163
164
  inventory.save!(inventory.sites_file)
164
- elsif new_sites.size>0
165
+ else
165
166
  inventory.save!
166
167
  end
167
168
  inventory=nil
168
- puts "Done! New found sites are successfully saved. " if new_sites.size > 0
169
+ puts "Done! New found sites are successfully saved. " unless new_sites.nil? or new_sites.empty?
169
170
  end
170
171
 
171
172
 
172
173
  # seventh step - update the hosts repository
174
+ puts "Invoke the HostTracker with optional directory setter."
175
+ host_tracker = Wmap::HostTracker.instance
173
176
  if options[:target] && options[:data_dir]
174
- puts "Invoke the HostTracker with optional directory setter."
175
- host_tracker = Wmap::HostTracker.instance
177
+ puts puts "Invoke the HostTracker with options: #{options[:data_dir]}, #{options[:target]}"
176
178
  host_tracker.verbose=options[:verbose]
177
179
  host_tracker.data_dir = options[:data_dir]
178
180
  host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
179
181
  host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
180
182
  elsif options[:target]
181
- puts puts "Invoke the HostTracker."
182
- host_tracker = Wmap::HostTracker.instance
183
- host_tracker.verbose=options[:verbose]
183
+ puts puts "Invoke the HostTracker with option: #{options[:target]}."
184
+ #host_tracker.verbose=options[:verbose]
184
185
  else
185
186
  abort "Error firing up HostTracker instance!"
186
187
  end
@@ -18,8 +18,8 @@ class Wmap::CidrTracker
18
18
  @verbose=params.fetch(:verbose, false)
19
19
  @data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
20
20
  Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
21
- @cidr_seeds=params.fetch(:cidr_seeds, @data_dir + 'cidrs')
22
- File.write(@cidr_seeds, "") unless File.exist?(@cidr_seeds)
21
+ @cidr_seeds=params.fetch(:cidr_seeds, @data_dir + '/' + 'cidrs')
22
+ File.new(@cidr_seeds, "w") unless File.exist?(@cidr_seeds)
23
23
  load_cidr_blks_from_file(@cidr_seeds)
24
24
  end
25
25
 
@@ -48,7 +48,7 @@ class Wmap::CidrTracker
48
48
  # 'setter' to load the known CIDR blocks into an instance variable @known_cidr_blks
49
49
  def load_cidr_blks_from_file(file_cidrs=@cidr_seeds)
50
50
  puts "Load the known CIDR seed file: #{file_cidrs}" if @verbose
51
- f=File.open(file_cidrs, 'r')
51
+ f=File.open(file_cidrs, 'r', :encoding => 'UTF-8')
52
52
  f.each do |line|
53
53
  entry=line.chomp.split(',')
54
54
  next unless is_cidr?(entry[0])
@@ -27,7 +27,7 @@ class SubDomain < Wmap::DomainTracker
27
27
  @max_parallel=params.fetch(:max_parallel, 40)
28
28
  # Hash table to hold the trusted domains
29
29
  @sub_domains_file=params.fetch(:sub_domains_file, @data_dir + 'sub_domains')
30
- File.write(@sub_domains_file, "") unless File.exist?(@sub_domains_file)
30
+ File.new(@sub_domains_file, "w") unless File.exist?(@sub_domains_file)
31
31
  @known_internet_sub_domains=load_domains_from_file(@sub_domains_file) #unless @known_internet_sub_domains.size>0
32
32
  end
33
33
 
@@ -70,7 +70,7 @@ class SubDomain < Wmap::DomainTracker
70
70
  end
71
71
  end
72
72
  @known_internet_sub_domains.merge!(results)
73
- puts "Done loading entries."
73
+ puts "Done loading sub_domain entries."
74
74
  return results
75
75
  else
76
76
  puts "Error: no entry is loaded. Please check your list and try again."
@@ -25,7 +25,7 @@ class Wmap::DomainTracker
25
25
  @domains_file=params.fetch(:domains_file, @data_dir+'domains')
26
26
  @max_parallel=params.fetch(:max_parallel, 40)
27
27
  # Hash table to hold the trusted domains
28
- File.write(@domains_file, "") unless File.exist?(@domains_file)
28
+ File.new(@domains_file, "w") unless File.exist?(@domains_file)
29
29
  load_domains_from_file(@domains_file)
30
30
  end
31
31
 
@@ -169,7 +169,7 @@ class Wmap::DomainTracker
169
169
  end
170
170
  end
171
171
  @known_internet_domains.merge!(results)
172
- puts "Done loading entries."
172
+ puts "Done loading domain entries."
173
173
  return results
174
174
  else
175
175
  puts "Error: no entry is loaded. Please check your list and try again."
@@ -262,7 +262,7 @@ class Wmap::DomainTracker
262
262
  when "Wmap::DomainTracker::SubDomain"
263
263
  return @known_internet_sub_domains.key?(domain)
264
264
  else
265
- return nil
265
+ return false
266
266
  end
267
267
  rescue => ee
268
268
  puts "Exception on method #{__method__}: #{ee}" if @verbose
@@ -21,11 +21,11 @@ module Wmap
21
21
  # Initialize the instance variables
22
22
  def initialize (params = {})
23
23
  @verbose=params.fetch(:verbose, false)
24
- @data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../../data/')
24
+ @data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../../data/')
25
25
  # Set default instance variables
26
26
  @hosts_file=params.fetch(:hosts_file, @data_dir + 'prime_hosts')
27
27
  # Initialize the instance variables
28
- File.write(@hosts_file, "") unless File.exist?(@hosts_file)
28
+ File.new(@hosts_file, "w") unless File.exist?(@hosts_file)
29
29
  @known_hosts=load_known_hosts_from_file(@hosts_file)
30
30
  @known_ips=Hash.new
31
31
  de_duplicate
@@ -26,16 +26,16 @@ class Wmap::HostTracker
26
26
  @hosts_file=params.fetch(:hosts_file, @data_dir + 'hosts')
27
27
  @max_parallel=params.fetch(:max_parallel, 40)
28
28
  # Initialize the instance variables
29
- File.write(@hosts_file, "") unless File.exist?(@hosts_file)
30
- load_known_hosts_from_file(@hosts_file)
29
+ File.new(@hosts_file, "w") unless File.exist?(@hosts_file)
30
+ @known_hosts=load_known_hosts_from_file(@hosts_file)
31
31
  end
32
32
 
33
33
  # Setter to load the known hosts from the local hosts file into a class instance
34
34
  def load_known_hosts_from_file (f_hosts=@hosts_file)
35
35
  puts "Loading local hosts from file: #{f_hosts} ..." if @verbose
36
- @known_hosts=Hash.new
36
+ known_hosts=Hash.new
37
37
  @alias = Hash.new
38
- File.write(f_hosts, "") unless File.exist?(f_hosts)
38
+ File.new(f_hosts, "w") unless File.exist?(f_hosts)
39
39
  f=File.open(f_hosts, 'r')
40
40
  f.each do |line|
41
41
  next unless line =~ /\d+\.\d+\.\d+\.\d+/
@@ -43,11 +43,11 @@ class Wmap::HostTracker
43
43
  key=entry[0].downcase
44
44
  value=entry[1]
45
45
  puts "Loading key value pair: #{key} - #{value}" if @verbose
46
- @known_hosts[key] = Hash.new unless @known_hosts.key?(key)
47
- @known_hosts[key]= value
46
+ known_hosts[key] = Hash.new unless known_hosts.key?(key)
47
+ known_hosts[key]= value
48
48
  # For reverse host lookup
49
- @known_hosts[value] = Hash.new unless @known_hosts.key?(value)
50
- @known_hosts[value] = key
49
+ known_hosts[value] = Hash.new unless known_hosts.key?(value)
50
+ known_hosts[value] = key
51
51
  # Count the number of alias for the recorded IP
52
52
  if @alias.key?(value)
53
53
  @alias[value]+=1
@@ -56,11 +56,12 @@ class Wmap::HostTracker
56
56
  end
57
57
  end
58
58
  f.close
59
- return @known_hosts
60
- rescue => ee
61
- puts "Exception on method #{__method__}: #{ee}"
62
59
  return known_hosts
60
+ #rescue => ee
61
+ # puts "Exception on method #{__method__}: #{ee}"
62
+ # return known_hosts
63
63
  end
64
+ alias_method :load, :load_known_hosts_from_file
64
65
 
65
66
  # Save the current local hosts hash table into a (random) data repository file
66
67
  def save_known_hosts_to_file!(f_hosts=@hosts_file)
@@ -96,30 +97,42 @@ class Wmap::HostTracker
96
97
  puts "Exception on method #{__method__}: #{ee}"
97
98
  end
98
99
 
100
+ # determine if host is part of trusted (known) root domains
101
+ def is_trusted?(host)
102
+ puts "Determin if host #{host} is part of trusted root domains" if @verbose
103
+ root=get_domain_root(host)
104
+ puts "Domain root: #{root}" if @verbose
105
+ domain_tracker=Wmap::DomainTracker.instance
106
+ domain_tracker.data_dir=@data_dir
107
+ domain_tracker.domains_file = domain_tracker.data_dir + "/" + "domains"
108
+ domain_tracker.load_domains_from_file
109
+ if domain_tracker.domain_known?(root)
110
+ domain_tracker=nil
111
+ return true
112
+ else
113
+ domain_tracker=nil
114
+ return false
115
+ end
116
+ end
117
+
99
118
  # Setter to add host entry to the cache once at a time
100
119
  def add(host)
101
120
  puts "Add entry to the local host repository: #{host}"
102
121
  host=host.strip.downcase unless host.nil?
122
+ root=get_domain_root(host)
103
123
  unless @known_hosts.key?(host)
104
124
  ip=host_2_ip(host)
105
125
  record=Hash.new
106
126
  if is_ip?(ip)
107
127
  # filter host to known domains only
108
- root=get_domain_root(host)
109
- puts "Domain root: #{root}" if @verbose
110
- domain_tracker=Wmap::DomainTracker.instance
111
- domain_tracker.data_dir=@data_dir
112
- domain_tracker.domains_file = domain_tracker.data_dir + "domains"
113
- domain_tracker.load_domains_from_file
114
- if domain_tracker.domain_known?(root)
115
- domain_tracker=nil
128
+ if is_trusted?(host)
116
129
  record[host]=ip
117
130
  record[ip]=host
118
131
  puts "Host data repository entry loaded: #{host} <=> #{ip}"
119
132
  # Replace instance with the class variable to avoid potential race condition under parallel engine
120
133
  # add additional logic to update the sub-domain table as well, 02/10/2014
121
134
  sub=get_sub_domain(host)
122
- if sub!=root
135
+ if sub!=nil
123
136
  tracker=Wmap::DomainTracker::SubDomain.instance
124
137
  tracker.data_dir=@data_dir
125
138
  tracker.sub_domains_file = tracker.data_dir + "sub_domains"
@@ -142,8 +155,8 @@ class Wmap::HostTracker
142
155
  else
143
156
  puts "Host is already exist. Skip: #{host}"
144
157
  end
145
- rescue => ee
146
- puts "Exception on method #{__method__}: #{ee}" if @verbose
158
+ #rescue => ee
159
+ # puts "Exception on method #{__method__}: #{ee}" if @verbose
147
160
  end
148
161
 
149
162
  # Setter to add host entry to the local hosts in batch (from an array)
@@ -164,7 +177,7 @@ class Wmap::HostTracker
164
177
  end
165
178
  end
166
179
  @known_hosts.merge!(results)
167
- puts "Done loading entries."
180
+ puts "Done loading host entries."
168
181
  return results
169
182
  else
170
183
  puts "Error: empty list - no entry is loaded. Please check your input list and try again."
@@ -27,7 +27,7 @@ class DeactivatedSite < Wmap::SiteTracker
27
27
  @sites_file=params.fetch(:sites_file, @data_dir + 'deactivated_sites')
28
28
  @verbose=params.fetch(:verbose, false)
29
29
  # Hash table to hold the site store
30
- File.write(@sites_file, "") unless File.exist?(@sites_file)
30
+ File.new(@sites_file, "w") unless File.exist?(@sites_file)
31
31
  @known_sites=load_site_stores_from_file(@sites_file)
32
32
  end
33
33
 
@@ -25,13 +25,13 @@ class WpTracker < Wmap::SiteTracker
25
25
  @verbose=params.fetch(:verbose, false)
26
26
  @data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../../data/')
27
27
  Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
28
- @sites_wp=params.fetch(:sites_wp, @data_dir+"wp_sites")
28
+ @sites_wp=params.fetch(:sites_wp, @data_dir+"wp_sites")
29
29
  @http_timeout=params.fetch(:http_timeout, 5000)
30
30
  @max_parallel=params.fetch(:max_parallel, 40)
31
31
  Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
32
32
  @log_file=@data_dir + "wp_checker.log"
33
- File.write(@sites_wp, "") unless File.exist?(@sites_wp)
34
- load_from_file(@sites_wp)
33
+ File.new(@sites_wp, "w") unless File.exist?(@sites_wp)
34
+ load_from_file(@sites_wp)
35
35
  end
36
36
 
37
37
  # 'setter' to load the known wordpress sites into an instance variable
@@ -174,7 +174,7 @@ class WpTracker < Wmap::SiteTracker
174
174
  end
175
175
  end
176
176
  @known_wp_sites.merge!(results)
177
- puts "Done loading entries."
177
+ puts "Done loading wp entries."
178
178
  return results
179
179
  else
180
180
  puts "Error: no entry is loaded. Please check your list and try again."
@@ -25,7 +25,7 @@ class Wmap::SiteTracker
25
25
  @sites_file=params.fetch(:sites_file, @data_dir+'sites')
26
26
  @verbose=params.fetch(:verbose, false)
27
27
  @max_parallel=params.fetch(:max_parallel, 30)
28
- File.write(@sites_file, "") unless File.exist?(@sites_file)
28
+ File.new(@sites_file, "w") unless File.exist?(@sites_file)
29
29
  # Hash table to hold the site store
30
30
  load_site_stores_from_file(@sites_file)
31
31
  end
@@ -34,7 +34,7 @@ class Wmap::SiteTracker
34
34
  def load_site_stores_from_file (file=@sites_file)
35
35
  puts "Loading the site store data repository from file: #{file} " if @verbose
36
36
  @known_sites=Hash.new
37
- File.write(file, "") unless File.exist?(file)
37
+ File.new(file, "w") unless File.exist?(file)
38
38
  f=File.open(file, 'r')
39
39
  f.each do |line|
40
40
  line=line.chomp.strip
@@ -77,7 +77,9 @@ class Wmap::SiteTracker
77
77
  f.write "# Local site store created by class #{self.class} method #{__method__} at: #{timestamp}\n"
78
78
  f.write "# Website,Primary IP,Port,Hosting Status,Server,Response Code,MD5 Finger-print,Redirection,Timestamp\n"
79
79
  @known_sites.keys.sort.map do |key|
80
- f.write "#{key},#{@known_sites[key]['ip']},#{@known_sites[key]['port']},#{@known_sites[key]['status']},#{@known_sites[key]['server']},#{@known_sites[key]['code']},#{@known_sites[key]['md5']},#{@known_sites[key]['redirection']},#{@known_sites[key]['timestamp']}\n"
80
+ if is_trusted?(key)
81
+ f.write "#{key},#{@known_sites[key]['ip']},#{@known_sites[key]['port']},#{@known_sites[key]['status']},#{@known_sites[key]['server']},#{@known_sites[key]['code']},#{@known_sites[key]['md5']},#{@known_sites[key]['redirection']},#{@known_sites[key]['timestamp']}\n"
82
+ end
81
83
  end
82
84
  f.close
83
85
  puts "site store table is successfully saved: #{file_sites}"
@@ -94,6 +96,24 @@ class Wmap::SiteTracker
94
96
  puts "Exception on method #{__method__}: #{ee}"
95
97
  end
96
98
 
99
+ # determine site is trusted based on the known domains
100
+ def is_trusted?(site)
101
+ trusted=false
102
+ host=url_2_host(site)
103
+ root=get_domain_root(host)
104
+ domain_tracker=Wmap::DomainTracker.instance
105
+ domain_tracker.data_dir=@data_dir
106
+ domain_tracker.domains_file=@data_dir + "/" + "domains"
107
+ File.new(domain_tracker.domains_file, "w") unless File.exist?(domain_tracker.domains_file)
108
+ domain_tracker.load_domains_from_file(domain_tracker.domains_file)
109
+ trusted=domain_tracker.domain_known?(root)
110
+ domain_tracker=nil
111
+ return trusted
112
+ rescue => ee
113
+ puts "Exception on method #{__method__}: #{ee}"
114
+ return trusted
115
+ end
116
+
97
117
  # Setter to add site entry to the cache one at a time
98
118
  def add(site)
99
119
  puts "Add entry to the site store: #{site}"
@@ -113,7 +133,7 @@ class Wmap::SiteTracker
113
133
  # Additional logic to refresh deactivated site, 02/12/2014
114
134
  deact=Wmap::SiteTracker::DeactivatedSite.instance
115
135
  deact.sites_file=@data_dir + "/" + "deactivated_sites"
116
- File.write(deact.sites_file, "") unless File.exist?(deact.sites_file)
136
+ File.new(deact.sites_file, "w") unless File.exist?(deact.sites_file)
117
137
  deact.load_site_stores_from_file
118
138
  # only trust either the domain or IP we know
119
139
  if is_ip?(host)
@@ -125,13 +145,17 @@ class Wmap::SiteTracker
125
145
  else
126
146
  domain_tracker=Wmap::DomainTracker.instance
127
147
  domain_tracker.domains_file=@data_dir + "/" + "domains"
128
- File.write(domain_tracker.domains_file, "") unless File.exist?(domain_tracker.domains_file)
148
+ File.new(domain_tracker.domains_file, "w") unless File.exist?(domain_tracker.domains_file)
129
149
  domain_tracker.load_domains_from_file(domain_tracker.domains_file)
130
150
  trusted=domain_tracker.domain_known?(root)
131
151
  domain_tracker=nil
132
152
  end
133
153
  end
134
154
  # add record only if trusted
155
+ host_tracker = Wmap::HostTracker.instance
156
+ host_tracker.data_dir= @data_dir
157
+ host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
158
+ host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
135
159
  if trusted
136
160
  # Add logic to check site status before adding it
137
161
  checker=Wmap::UrlChecker.new(:data_dir=>@data_dir).check(site)
@@ -144,10 +168,6 @@ class Wmap::SiteTracker
144
168
  raise "Site is currently down. Skip #{site}" if checker['code']==10000
145
169
  end
146
170
  raise "Exception on add method - Fail to resolve the host-name: Host - #{host}, IP - #{ip}. Skip #{site}" unless is_ip?(ip)
147
- host_tracker = Wmap::HostTracker.instance
148
- host_tracker.data_dir= @data_dir
149
- host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
150
- host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
151
171
  # Update the local host table when necessary
152
172
  if is_ip?(host)
153
173
  # Case #1: Trusted site contains IP
@@ -291,7 +311,7 @@ class Wmap::SiteTracker
291
311
  # Additional logic to deactivate the site properly, by moving it to the DeactivatedSite list, 02/07/2014
292
312
  deact=Wmap::SiteTracker::DeactivatedSite.instance
293
313
  deact.sites_file=@data_dir + 'deactivated_sites'
294
- File.write(deact.sites_file, "") unless File.exist?(deact.sites_file)
314
+ File.new(deact.sites_file, "w") unless File.exist?(deact.sites_file)
295
315
  site=site.strip.downcase
296
316
  site=url_2_site(site)
297
317
  if @known_sites.key?(site)
@@ -341,8 +361,8 @@ class Wmap::SiteTracker
341
361
  else
342
362
  puts "Error: no entry is loaded. Please check your list and try again."
343
363
  end
344
- #rescue => ee
345
- # puts "Exception on method #{__method__}: #{ee}" if @verbose
364
+ rescue => ee
365
+ puts "Exception on method #{__method__}: #{ee}" if @verbose
346
366
  end
347
367
  alias_method :dels, :bulk_delete
348
368
 
@@ -88,9 +88,9 @@ class Wmap::UrlChecker
88
88
  checker['redirection']=nil
89
89
  checker['timestamp']=timestamp
90
90
  return checker
91
- rescue Exception => ee
92
- puts "Exception on method #{__method__} for #{url}: #{ee}" # if @verbose
93
- return nil
91
+ #rescue Exception => ee
92
+ # puts "Exception on method #{__method__} for #{url}: #{ee}" # if @verbose
93
+ # return nil
94
94
  end
95
95
  alias_method :check, :url_worker
96
96
 
@@ -28,7 +28,7 @@ module Wmap
28
28
  file=params.fetch(:signature_file, @signature_file)
29
29
  @tag_signatures=load_sig_from_file(file)
30
30
  @tag_file=params.fetch(:tag_file, @data_dir + 'tag_sites')
31
- File.write(@tag_file, "") unless File.exist?(@tag_file)
31
+ File.new(@tag_file, "w") unless File.exist?(@tag_file)
32
32
  # load the known tag store
33
33
  load_tag_from_file(@tag_file)
34
34
  @landings = Hash.new # cache landing page to reduce redundant browsing
@@ -119,7 +119,7 @@ module Wmap
119
119
  end
120
120
  end
121
121
  @tag_store.merge!(results)
122
- puts "Done loading entries."
122
+ puts "Done loading adware entries."
123
123
  tags = nil
124
124
  return results
125
125
  else
@@ -90,7 +90,7 @@ class Wmap::UrlCrawler
90
90
 
91
91
  # The worker instance of crawler who perform the labour work
92
92
  def crawl_worker(url0)
93
- puts "Please be aware that it may take a while to crawl #{url0}, depending on the site's responsiveness and the amount of contents."
93
+ puts "Please be aware that it may take a while to crawl #{url0}, depending on the site's responsiveness and discovery contents."
94
94
  # Input URL sanity check first
95
95
  if is_url?(url0)
96
96
  host=url_2_host(url0)
@@ -1,20 +1,15 @@
1
1
  # Default list of Google search keywords (known company legal names, for example) for the service discovery, one entry at per line
2
2
  # Used by the Wmap::GoogleSearchScraper class
3
+ Google llc
3
4
 
4
5
  # Official whole name
5
- Penguin Random House
6
6
 
7
7
  # 2014 purchase
8
- Random House
9
8
 
10
9
  # Parent company
11
- Bertelsmann
12
10
 
13
11
  # Division
14
- Crown Books
15
12
 
16
13
  # Division
17
- Dorling Kindersley
18
14
 
19
15
  # Division
20
- Penguin Random House Canada
data/version.txt CHANGED
@@ -3,8 +3,8 @@
3
3
  ###############################################################################
4
4
  package = wmap
5
5
  # wmap version 2.0 == web_discovery version 1.5.3
6
- version = 2.8.2
7
- date = 2021-07-23
6
+ version = 2.8.4
7
+ date = 2024-07-01
8
8
 
9
9
  author = Sam (Yang) Li
10
10
  email = yang.li@owasp.org
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: wmap
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.8.2
4
+ version: 2.8.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sam (Yang) Li
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-07-23 00:00:00.000000000 Z
11
+ date: 2024-07-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: dnsruby