wmap 2.5.5 → 2.5.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -14,9 +14,7 @@ class Wmap::DomainTracker
14
14
  include Wmap::Utils
15
15
  include Singleton
16
16
 
17
-
18
- attr_accessor :verbose, :max_parallel, :domains_file, :file_domains, :data_dir
19
- attr_reader :known_internet_domains
17
+ attr_accessor :verbose, :max_parallel, :domains_file, :data_dir, :known_internet_domains
20
18
 
21
19
  # Set default instance variables
22
20
  def initialize (params = {})
@@ -24,275 +22,250 @@ class Wmap::DomainTracker
24
22
  @verbose=params.fetch(:verbose, false)
25
23
  @data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
26
24
  Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
27
- @file_domains=params.fetch(:domains_file, @data_dir+'domains')
25
+ @domains_file=params.fetch(:domains_file, @data_dir+'domains')
28
26
  @max_parallel=params.fetch(:max_parallel, 40)
29
27
  # Hash table to hold the trusted domains
30
- File.write(@file_domains, "") unless File.exist?(@file_domains)
31
- @known_internet_domains=load_domains_from_file(@file_domains)
32
- #@known_internet_sub_domains=Hash.new
28
+ File.write(@domains_file, "") unless File.exist?(@domains_file)
29
+ load_domains_from_file(@domains_file)
33
30
  end
34
31
 
35
32
  # 'setter' to load the known Internet domains into an instance variable
36
- def load_domains_from_file (file=@file_domains, lc=true)
33
+ def load_domains_from_file (file=@domains_file, lc=true)
37
34
  puts "Loading trusted domain file: #{file}" if @verbose
38
- begin
39
- known_internet_domains=Hash.new
40
- f_domains=File.open(file, 'r')
41
- f_domains.each_line do |line|
42
- puts "Processing line: #{line}" if @verbose
43
- line=line.chomp.strip
44
- next if line.nil?
45
- next if line.empty?
46
- next if line =~ /^\s*#/
47
- line=line.downcase if lc==true
48
- entry=line.split(',')
49
- if known_internet_domains.key?(entry[0])
50
- next
35
+ @known_internet_domains=Hash.new
36
+ f_domains=File.open(file, 'r')
37
+ f_domains.each_line do |line|
38
+ puts "Processing line: #{line}" if @verbose
39
+ line=line.chomp.strip
40
+ next if line.nil?
41
+ next if line.empty?
42
+ next if line =~ /^\s*#/
43
+ line=line.downcase if lc==true
44
+ entry=line.split(',')
45
+ if @known_internet_domains.key?(entry[0])
46
+ next
47
+ else
48
+ if entry[1] =~ /yes/i
49
+ @known_internet_domains[entry[0]]=true
51
50
  else
52
- if entry[1] =~ /yes/i
53
- known_internet_domains[entry[0]]=true
54
- else
55
- known_internet_domains[entry[0]]=false
56
- end
51
+ @known_internet_domains[entry[0]]=false
57
52
  end
58
-
59
53
  end
60
- f_domains.close
61
- return known_internet_domains
62
- rescue => ee
63
- puts "Exception on method #{__method__}: #{ee}" if @verbose
64
- return nil
54
+
65
55
  end
56
+ f_domains.close
57
+ return @known_internet_domains
58
+ rescue => ee
59
+ puts "Exception on method #{__method__}: #{ee}" if @verbose
60
+ return nil
66
61
  end
67
62
 
68
63
  # Save the current domain hash table into a file
69
- def save_domains_to_file!(file_domains=@file_domains, domains=@known_internet_domains)
64
+ def save_domains_to_file!(file_domains=@domains_file, domains=@known_internet_domains)
70
65
  puts "Saving the current domains cache table from memory to file: #{file_domains} ..." if @verbose
71
- begin
72
- timestamp=Time.now
73
- f=File.open(file_domains, 'w')
74
- f.write "# Local domains file created by class #{self.class} method #{__method__} at: #{timestamp}\n"
75
- f.write "# domain name, free zone transfer detected?\n"
76
- domains.keys.sort.map do |key|
77
- if domains[key]
78
- f.write "#{key}, yes\n"
79
- else
80
- f.write "#{key}, no\n"
81
- end
66
+ timestamp=Time.now
67
+ f=File.open(file_domains, 'w')
68
+ f.write "# Local domains file created by class #{self.class} method #{__method__} at: #{timestamp}\n"
69
+ f.write "# domain name, free zone transfer detected?\n"
70
+ domains.keys.sort.map do |key|
71
+ if domains[key]
72
+ f.write "#{key}, yes\n"
73
+ else
74
+ f.write "#{key}, no\n"
82
75
  end
83
- f.close
84
- puts "Domain cache table is successfully saved: #{file_domains}"
85
- rescue => ee
86
- puts "Exception on method #{__method__}: #{ee}" if @verbose
87
76
  end
77
+ f.close
78
+ puts "Domain cache table is successfully saved: #{file_domains}"
79
+ rescue => ee
80
+ puts "Exception on method #{__method__}: #{ee}" if @verbose
88
81
  end
89
82
  alias_method :save!, :save_domains_to_file!
90
83
 
91
84
  # Count numbers of entries in the domain cache table
92
85
  def count
93
86
  puts "Counting number of entries in the domain cache table ..."
94
- begin
95
- cnt=0
96
- @known_internet_domains.map do |key|
97
- unless key =~ /\w+\.\w+/
98
- cnt=cnt+1
99
- end
87
+ cnt=0
88
+ @known_internet_domains.map do |key|
89
+ unless key =~ /\w+\.\w+/
90
+ cnt=cnt+1
100
91
  end
101
- puts "Current number of entries: #{cnt}"
102
- return cnt
103
- rescue => ee
104
- puts "Exception on method #{__method__}: #{ee}" if @verbose
105
92
  end
93
+ puts "Current number of entries: #{cnt}"
94
+ return cnt
95
+ rescue => ee
96
+ puts "Exception on method #{__method__}: #{ee}" if @verbose
106
97
  end
107
98
  alias_method :size, :count
108
99
 
109
100
  # 'setter' to add domain entry to the cache one at a time
110
101
  def add(host)
111
- begin
112
- puts "Add entry to the local domains cache table: #{host}" if @verbose
113
- return nil if host.nil? or host.empty?
114
- host=host.strip.downcase
115
- if @known_internet_domains.key?(host)
116
- puts "Domain is already exist. Skipping: #{host}"
117
- else
118
- root=get_domain_root(host)
119
- sub=get_subdomain(host)
120
- record=Hash.new
121
- if host == root
122
- if zone_transferable?(root)
123
- record[root]=true
124
- #@known_internet_domains[root]=true
125
- else
126
- record[root]=false
127
- #@known_internet_domains[root]=false
128
- end
129
- puts "Entry loaded: #{record}"
130
- @known_internet_domains.merge!(record)
131
- return record
132
- elsif sub.nil? # 2/10/2014, additional logic to support sub-domains
133
- # do nothing
134
- elsif host != sub
135
- if zone_transferable?(sub)
136
- #@known_internet_domains[sub]=true
137
- record[sub]=true
138
- else
139
- #@known_internet_domains[sub]=false
140
- record[sub]=false
141
- end
142
- puts "Entry loaded: #{record}"
143
- @known_internet_domains.merge!(record)
144
- return record
102
+ puts "Add entry to the local domains cache table: #{host}" if @verbose
103
+ return nil if host.nil? or host.empty?
104
+ host=host.strip.downcase
105
+ if @known_internet_domains.key?(host)
106
+ puts "Domain is already exist. Skipping: #{host}"
107
+ else
108
+ root=get_domain_root(host)
109
+ sub=get_subdomain(host)
110
+ record=Hash.new
111
+ if host == root
112
+ if zone_transferable?(root)
113
+ record[root]=true
114
+ #@known_internet_domains[root]=true
115
+ else
116
+ record[root]=false
117
+ #@known_internet_domains[root]=false
118
+ end
119
+ puts "Entry loaded: #{record}"
120
+ @known_internet_domains.merge!(record)
121
+ return record
122
+ elsif sub.nil? # 2/10/2014, additional logic to support sub-domains
123
+ # do nothing
124
+ elsif host != sub
125
+ if zone_transferable?(sub)
126
+ #@known_internet_domains[sub]=true
127
+ record[sub]=true
145
128
  else
146
- puts "Problem add domain #{host}: unknown domain format - please use legal root domain or sub domain only."
129
+ #@known_internet_domains[sub]=false
130
+ record[sub]=false
147
131
  end
132
+ puts "Entry loaded: #{record}"
133
+ @known_internet_domains.merge!(record)
134
+ return record
135
+ else
136
+ puts "Problem add domain #{host}: unknown domain format - please use legal root domain or sub domain only."
148
137
  end
149
- rescue => ee
150
- puts "Exception on method #{__method__}: #{ee}" if @verbose
151
138
  end
139
+ rescue => ee
140
+ puts "Exception on method #{__method__}: #{ee}" if @verbose
152
141
  end
153
142
 
154
143
  # 'setter' to add domain entry to the cache in batch (from a file)
155
144
  def file_add(file)
156
- begin
157
- puts "Add entries to the local domains cache table from file: #{file}" if @verbose
158
- raise "File non-exist. Please check your file path and name again: #{file}" unless File.exist?(file)
159
- changes=Array.new
160
- domains=file_2_list(file)
161
- changes=bulk_add(domains)
162
- rescue => ee
163
- puts "Exception on method #{__method__}: #{ee}" if @verbose
164
- end
145
+ puts "Add entries to the local domains cache table from file: #{file}" if @verbose
146
+ raise "File non-exist. Please check your file path and name again: #{file}" unless File.exist?(file)
147
+ changes=Array.new
148
+ domains=file_2_list(file)
149
+ changes=bulk_add(domains)
150
+ rescue => ee
151
+ puts "Exception on method #{__method__}: #{ee}" if @verbose
165
152
  end
166
153
 
167
154
  # 'setter' to add domain entry to the cache in batch (from a list)
168
155
  def bulk_add(list, num=@max_parallel)
169
156
  puts "Add entries to the local domains cache table from list: #{list}" if @verbose
170
- begin
171
- results=Hash.new
172
- domains=list
173
- if domains.size > 0
174
- Parallel.map(list, :in_processes => num) { |target|
175
- add(target)
176
- }.each do |process|
177
- if process.nil?
178
- next
179
- elsif process.empty?
180
- #do nothing
181
- else
182
- results.merge!(process)
183
- end
157
+ results=Hash.new
158
+ domains=list
159
+ if domains.size > 0
160
+ Parallel.map(list, :in_processes => num) { |target|
161
+ add(target)
162
+ }.each do |process|
163
+ if process.nil?
164
+ next
165
+ elsif process.empty?
166
+ #do nothing
167
+ else
168
+ results.merge!(process)
184
169
  end
185
- @known_internet_domains.merge!(results)
186
- puts "Done loading entries."
187
- return results
188
- else
189
- puts "Error: no entry is loaded. Please check your list and try again."
190
170
  end
171
+ @known_internet_domains.merge!(results)
172
+ puts "Done loading entries."
191
173
  return results
192
- rescue => ee
193
- puts "Exception on method #{__method__}: #{ee}" if @verbose
174
+ else
175
+ puts "Error: no entry is loaded. Please check your list and try again."
194
176
  end
177
+ return results
178
+ rescue => ee
179
+ puts "Exception on method #{__method__}: #{ee}" if @verbose
195
180
  end
196
181
  alias_method :adds, :bulk_add
197
182
 
198
183
  # 'setter' to remove entry from the cache one at a time
199
184
  def delete(domain)
200
185
  puts "Remove entry from the domains cache table: #{domain} " if @verbose
201
- begin
202
- domain=domain.strip.downcase
203
- if @known_internet_domains.key?(domain)
204
- @known_internet_domains.delete(domain)
205
- puts "Entry cleared: #{domain}"
206
- return domain
207
- else
208
- puts "Entry not fund. Skipping: #{domain}"
209
- end
210
- rescue => ee
211
- puts "Exception on method #{__method__}: #{ee}" if @verbose
186
+ domain=domain.strip.downcase
187
+ if @known_internet_domains.key?(domain)
188
+ @known_internet_domains.delete(domain)
189
+ puts "Entry cleared: #{domain}"
190
+ return domain
191
+ else
192
+ puts "Entry not fund. Skipping: #{domain}"
212
193
  end
194
+ rescue => ee
195
+ puts "Exception on method #{__method__}: #{ee}" if @verbose
213
196
  end
214
197
 
215
198
  # 'setter' to delete domain entry to the cache in batch (from a list)
216
199
  def bulk_delete(list)
217
200
  puts "Delete entries to the local domains cache table from list: #{list}" if @verbose
218
- begin
219
- domains=list
220
- changes=Array.new
221
- if domains.size > 0
222
- domains.map do |x|
223
- domain=delete(x)
224
- changes.push(domain) unless domain.nil?
225
- end
226
- puts "Done deleting domains from list: #{list}"
227
- return changes
228
- else
229
- puts "Exception on method bulk_delete: no entry is loaded. Please check your list and try again."
201
+ domains=list
202
+ changes=Array.new
203
+ if domains.size > 0
204
+ domains.map do |x|
205
+ domain=delete(x)
206
+ changes.push(domain) unless domain.nil?
230
207
  end
231
- rescue => ee
232
- puts "Exception on method #{__method__}: #{ee}" if @verbose
208
+ puts "Done deleting domains from list: #{list}"
209
+ return changes
210
+ else
211
+ puts "Exception on method bulk_delete: no entry is loaded. Please check your list and try again."
233
212
  end
213
+ rescue => ee
214
+ puts "Exception on method #{__method__}: #{ee}" if @verbose
234
215
  end
235
216
  alias_method :dels, :bulk_delete
236
217
 
237
218
  # 'setter' to delete domain entry to the cache in batch (from a file)
238
219
  def file_delete(file)
239
- begin
240
- puts "Delete entries to the local domains cache table from file: #{file}" if @verbose
241
- raise "File non-exist. Please check your file path and name again: #{file}" unless File.exist?(file)
242
- domains=file_2_list(file)
243
- changes=bulk_delete(domains)
244
- rescue => ee
245
- puts "Exception on method #{__method__}: #{ee}" if @verbose
246
- end
220
+ puts "Delete entries to the local domains cache table from file: #{file}" if @verbose
221
+ raise "File non-exist. Please check your file path and name again: #{file}" unless File.exist?(file)
222
+ domains=file_2_list(file)
223
+ changes=bulk_delete(domains)
224
+ rescue => ee
225
+ puts "Exception on method #{__method__}: #{ee}" if @verbose
247
226
  end
248
227
 
249
228
  # 'setter' to remove all entries from the store
250
229
  def delete_all
251
230
  puts "Delete all entries in the domain store! " if @verbose
252
- begin
253
- @known_internet_domains.keys.map do |domain|
254
- delete(domain)
255
- end
256
- rescue => ee
257
- puts "Exception on method #{__method__}: #{ee}" if @verbose
231
+ @known_internet_domains.keys.map do |domain|
232
+ delete(domain)
258
233
  end
234
+ rescue => ee
235
+ puts "Exception on method #{__method__}: #{ee}" if @verbose
259
236
  end
260
237
 
261
238
  # Refresh the domain entry one at a time
262
239
  def refresh(domain)
263
- begin
264
- abort "Trusted Internet domain file not loaded properly! " if @known_internet_domains.nil?
265
- domain=domain.strip.downcase unless domain.nil?
266
- if domain_known?(domain)
267
- delete(domain)
268
- add(domain)
269
- return domain
270
- else
271
- puts "Unknown domain: #{domain}"
272
- return nil
273
- end
274
- rescue => ee
275
- puts "Exception on method #{__method__} for #{domain}: #{ee}" if @verbose
240
+ abort "Trusted Internet domain file not loaded properly! " if @known_internet_domains.nil?
241
+ domain=domain.strip.downcase unless domain.nil?
242
+ if domain_known?(domain)
243
+ delete(domain)
244
+ add(domain)
245
+ return domain
246
+ else
247
+ puts "Unknown domain: #{domain}"
276
248
  return nil
277
249
  end
250
+ rescue => ee
251
+ puts "Exception on method #{__method__} for #{domain}: #{ee}" if @verbose
252
+ return nil
278
253
  end
279
254
 
280
255
  # Simple method to check if a domain is already within the domain cache table
281
256
  def domain_known?(domain)
282
- begin
283
- #abort "Trusted Internet domain file not loaded properly! " if @known_internet_domains.nil? or @known_internet_sub_domains.nil?
284
- domain=domain.strip.downcase unless domain.nil?
285
- case self.class.name
286
- when "Wmap::DomainTracker"
287
- return @known_internet_domains.key?(domain)
288
- when "Wmap::DomainTracker::SubDomain"
289
- return @known_internet_sub_domains.key?(domain)
290
- else
291
- return nil
292
- end
293
- rescue => ee
294
- puts "Exception on method #{__method__}: #{ee}" if @verbose
257
+ #abort "Trusted Internet domain file not loaded properly! " if @known_internet_domains.nil? or @known_internet_sub_domains.nil?
258
+ domain=domain.strip.downcase unless domain.nil?
259
+ case self.class.name
260
+ when "Wmap::DomainTracker"
261
+ return @known_internet_domains.key?(domain)
262
+ when "Wmap::DomainTracker::SubDomain"
263
+ return @known_internet_sub_domains.key?(domain)
264
+ else
265
+ return nil
295
266
  end
267
+ rescue => ee
268
+ puts "Exception on method #{__method__}: #{ee}" if @verbose
296
269
  return false
297
270
  end
298
271
  alias_method :is_known?, :domain_known?
@@ -301,12 +274,10 @@ class Wmap::DomainTracker
301
274
  # Dump out the list of known domains
302
275
  def get_domains
303
276
  puts "Retrieve a list of known domain ..." if @verbose
304
- begin
305
- return @known_internet_domains.keys
306
- rescue Exception => ee
307
- puts "Exception on method #{__method__}: #{ee}" if @verbose
308
- return nil
309
- end
277
+ return @known_internet_domains.keys
278
+ rescue Exception => ee
279
+ puts "Exception on method #{__method__}: #{ee}" if @verbose
280
+ return nil
310
281
  end
311
282
  alias_method :dump_domains, :get_domains
312
283
  alias_method :dump, :get_domains
@@ -314,19 +285,17 @@ class Wmap::DomainTracker
314
285
  # Search potential matching domains from the domain store by using simple regular expression. Note that any upper-case char in the search string will be automatically converted into lower case
315
286
  def search (pattern)
316
287
  puts "Search domain store for the regular expression: #{pattern}" if @verbose
317
- begin
318
- pattern=pattern.strip.downcase
319
- results=Array.new
320
- @known_internet_domains.keys.map do |key|
321
- if key =~ /#{pattern}/i
322
- results.push(key)
323
- end
288
+ pattern=pattern.strip.downcase
289
+ results=Array.new
290
+ @known_internet_domains.keys.map do |key|
291
+ if key =~ /#{pattern}/i
292
+ results.push(key)
324
293
  end
325
- return results
326
- rescue Exception => ee
327
- puts "Exception on method #{__method__}: #{ee}" if @verbose
328
- return nil
329
294
  end
295
+ return results
296
+ rescue Exception => ee
297
+ puts "Exception on method #{__method__}: #{ee}" if @verbose
298
+ return nil
330
299
  end
331
300
  alias_method :find, :search
332
301
 
@@ -340,5 +309,4 @@ class Wmap::DomainTracker
340
309
  end
341
310
  alias_method :print, :print_known_domains
342
311
 
343
- private :load_domains_from_file
344
312
  end
@@ -16,19 +16,17 @@ module Wmap
16
16
  include Wmap::Utils
17
17
  include Singleton
18
18
 
19
- attr_accessor :hosts_file, :verbose, :data_dir
20
- attr_reader :known_hosts, :known_ips
19
+ attr_accessor :hosts_file, :verbose, :data_dir, :known_hosts, :known_ips
21
20
 
22
21
  # Initialize the instance variables
23
22
  def initialize (params = {})
24
23
  @verbose=params.fetch(:verbose, false)
25
24
  @data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../../data/')
26
25
  # Set default instance variables
27
- @file_hosts=@data_dir + 'prime_hosts'
28
- file=params.fetch(:hosts_file, @file_hosts)
26
+ @hosts_file=params.fetch(:hosts_file, @data_dir + 'prime_hosts')
29
27
  # Initialize the instance variables
30
- File.write(@file_hosts, "") unless File.exist?(@file_hosts)
31
- @known_hosts=load_known_hosts_from_file(file)
28
+ File.write(@hosts_file, "") unless File.exist?(@hosts_file)
29
+ @known_hosts=load_known_hosts_from_file(@hosts_file)
32
30
  @known_ips=Hash.new
33
31
  de_duplicate
34
32
  end
@@ -41,7 +39,8 @@ module Wmap
41
39
  cns=Hash.new
42
40
  checker=Wmap::UrlChecker.new(:data_dir=>@data_dir)
43
41
  my_tracker = Wmap::SiteTracker.instance
44
- my_tracker.data_dir = @data_dir
42
+ my_tracker.sites_file = @data_dir + "sites"
43
+ my_tracker.load_site_stores_from_file
45
44
  my_tracker.get_ssl_sites.map do |site|
46
45
  puts "Exam SSL enabled site entry #{site} ..."
47
46
  my_host=url_2_host(site)
@@ -77,9 +76,10 @@ module Wmap
77
76
  puts "Invoke internal procedures to update the primary host-name table from the site store."
78
77
  begin
79
78
  my_tracker=Wmap::SiteTracker.instance
80
- my_tracker.data_dir=@data_dir
79
+ my_tracker.sites_file=@data_dir + "sites"
80
+ my_tracker.load_site_stores_from_file
81
81
  urls = my_tracker.get_redirection_urls
82
- my_tracker = nil
82
+ my_tracker = nil
83
83
  urls.map do |url|
84
84
  if is_url?(url)
85
85
  host=url_2_host(url)