wmap 2.5.5 → 2.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/{README.rdoc → README.md} +47 -33
- data/bin/wmap +46 -32
- data/lib/wmap/domain_tracker/sub_domain.rb +67 -76
- data/lib/wmap/domain_tracker.rb +176 -208
- data/lib/wmap/host_tracker/primary_host.rb +9 -9
- data/lib/wmap/host_tracker.rb +314 -361
- data/lib/wmap/site_tracker/deactivated_site.rb +3 -4
- data/lib/wmap/site_tracker.rb +586 -640
- data/lib/wmap/utils/url_magic.rb +1 -1
- data/version.txt +2 -2
- data/wmap.gemspec +2 -2
- metadata +4 -5
- data/logs/wmap.log +0 -17
data/lib/wmap/domain_tracker.rb
CHANGED
@@ -14,9 +14,7 @@ class Wmap::DomainTracker
|
|
14
14
|
include Wmap::Utils
|
15
15
|
include Singleton
|
16
16
|
|
17
|
-
|
18
|
-
attr_accessor :verbose, :max_parallel, :domains_file, :file_domains, :data_dir
|
19
|
-
attr_reader :known_internet_domains
|
17
|
+
attr_accessor :verbose, :max_parallel, :domains_file, :data_dir, :known_internet_domains
|
20
18
|
|
21
19
|
# Set default instance variables
|
22
20
|
def initialize (params = {})
|
@@ -24,275 +22,250 @@ class Wmap::DomainTracker
|
|
24
22
|
@verbose=params.fetch(:verbose, false)
|
25
23
|
@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
|
26
24
|
Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
|
27
|
-
@
|
25
|
+
@domains_file=params.fetch(:domains_file, @data_dir+'domains')
|
28
26
|
@max_parallel=params.fetch(:max_parallel, 40)
|
29
27
|
# Hash table to hold the trusted domains
|
30
|
-
File.write(@
|
31
|
-
|
32
|
-
#@known_internet_sub_domains=Hash.new
|
28
|
+
File.write(@domains_file, "") unless File.exist?(@domains_file)
|
29
|
+
load_domains_from_file(@domains_file)
|
33
30
|
end
|
34
31
|
|
35
32
|
# 'setter' to load the known Internet domains into an instance variable
|
36
|
-
def load_domains_from_file (file=@
|
33
|
+
def load_domains_from_file (file=@domains_file, lc=true)
|
37
34
|
puts "Loading trusted domain file: #{file}" if @verbose
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
35
|
+
@known_internet_domains=Hash.new
|
36
|
+
f_domains=File.open(file, 'r')
|
37
|
+
f_domains.each_line do |line|
|
38
|
+
puts "Processing line: #{line}" if @verbose
|
39
|
+
line=line.chomp.strip
|
40
|
+
next if line.nil?
|
41
|
+
next if line.empty?
|
42
|
+
next if line =~ /^\s*#/
|
43
|
+
line=line.downcase if lc==true
|
44
|
+
entry=line.split(',')
|
45
|
+
if @known_internet_domains.key?(entry[0])
|
46
|
+
next
|
47
|
+
else
|
48
|
+
if entry[1] =~ /yes/i
|
49
|
+
@known_internet_domains[entry[0]]=true
|
51
50
|
else
|
52
|
-
|
53
|
-
known_internet_domains[entry[0]]=true
|
54
|
-
else
|
55
|
-
known_internet_domains[entry[0]]=false
|
56
|
-
end
|
51
|
+
@known_internet_domains[entry[0]]=false
|
57
52
|
end
|
58
|
-
|
59
53
|
end
|
60
|
-
|
61
|
-
return known_internet_domains
|
62
|
-
rescue => ee
|
63
|
-
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
64
|
-
return nil
|
54
|
+
|
65
55
|
end
|
56
|
+
f_domains.close
|
57
|
+
return @known_internet_domains
|
58
|
+
rescue => ee
|
59
|
+
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
60
|
+
return nil
|
66
61
|
end
|
67
62
|
|
68
63
|
# Save the current domain hash table into a file
|
69
|
-
def save_domains_to_file!(file_domains=@
|
64
|
+
def save_domains_to_file!(file_domains=@domains_file, domains=@known_internet_domains)
|
70
65
|
puts "Saving the current domains cache table from memory to file: #{file_domains} ..." if @verbose
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
domains
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
f.write "#{key}, no\n"
|
81
|
-
end
|
66
|
+
timestamp=Time.now
|
67
|
+
f=File.open(file_domains, 'w')
|
68
|
+
f.write "# Local domains file created by class #{self.class} method #{__method__} at: #{timestamp}\n"
|
69
|
+
f.write "# domain name, free zone transfer detected?\n"
|
70
|
+
domains.keys.sort.map do |key|
|
71
|
+
if domains[key]
|
72
|
+
f.write "#{key}, yes\n"
|
73
|
+
else
|
74
|
+
f.write "#{key}, no\n"
|
82
75
|
end
|
83
|
-
f.close
|
84
|
-
puts "Domain cache table is successfully saved: #{file_domains}"
|
85
|
-
rescue => ee
|
86
|
-
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
87
76
|
end
|
77
|
+
f.close
|
78
|
+
puts "Domain cache table is successfully saved: #{file_domains}"
|
79
|
+
rescue => ee
|
80
|
+
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
88
81
|
end
|
89
82
|
alias_method :save!, :save_domains_to_file!
|
90
83
|
|
91
84
|
# Count numbers of entries in the domain cache table
|
92
85
|
def count
|
93
86
|
puts "Counting number of entries in the domain cache table ..."
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
cnt=cnt+1
|
99
|
-
end
|
87
|
+
cnt=0
|
88
|
+
@known_internet_domains.map do |key|
|
89
|
+
unless key =~ /\w+\.\w+/
|
90
|
+
cnt=cnt+1
|
100
91
|
end
|
101
|
-
puts "Current number of entries: #{cnt}"
|
102
|
-
return cnt
|
103
|
-
rescue => ee
|
104
|
-
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
105
92
|
end
|
93
|
+
puts "Current number of entries: #{cnt}"
|
94
|
+
return cnt
|
95
|
+
rescue => ee
|
96
|
+
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
106
97
|
end
|
107
98
|
alias_method :size, :count
|
108
99
|
|
109
100
|
# 'setter' to add domain entry to the cache one at a time
|
110
101
|
def add(host)
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
if
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
record[sub]=true
|
138
|
-
else
|
139
|
-
#@known_internet_domains[sub]=false
|
140
|
-
record[sub]=false
|
141
|
-
end
|
142
|
-
puts "Entry loaded: #{record}"
|
143
|
-
@known_internet_domains.merge!(record)
|
144
|
-
return record
|
102
|
+
puts "Add entry to the local domains cache table: #{host}" if @verbose
|
103
|
+
return nil if host.nil? or host.empty?
|
104
|
+
host=host.strip.downcase
|
105
|
+
if @known_internet_domains.key?(host)
|
106
|
+
puts "Domain is already exist. Skipping: #{host}"
|
107
|
+
else
|
108
|
+
root=get_domain_root(host)
|
109
|
+
sub=get_subdomain(host)
|
110
|
+
record=Hash.new
|
111
|
+
if host == root
|
112
|
+
if zone_transferable?(root)
|
113
|
+
record[root]=true
|
114
|
+
#@known_internet_domains[root]=true
|
115
|
+
else
|
116
|
+
record[root]=false
|
117
|
+
#@known_internet_domains[root]=false
|
118
|
+
end
|
119
|
+
puts "Entry loaded: #{record}"
|
120
|
+
@known_internet_domains.merge!(record)
|
121
|
+
return record
|
122
|
+
elsif sub.nil? # 2/10/2014, additional logic to support sub-domains
|
123
|
+
# do nothing
|
124
|
+
elsif host != sub
|
125
|
+
if zone_transferable?(sub)
|
126
|
+
#@known_internet_domains[sub]=true
|
127
|
+
record[sub]=true
|
145
128
|
else
|
146
|
-
|
129
|
+
#@known_internet_domains[sub]=false
|
130
|
+
record[sub]=false
|
147
131
|
end
|
132
|
+
puts "Entry loaded: #{record}"
|
133
|
+
@known_internet_domains.merge!(record)
|
134
|
+
return record
|
135
|
+
else
|
136
|
+
puts "Problem add domain #{host}: unknown domain format - please use legal root domain or sub domain only."
|
148
137
|
end
|
149
|
-
rescue => ee
|
150
|
-
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
151
138
|
end
|
139
|
+
rescue => ee
|
140
|
+
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
152
141
|
end
|
153
142
|
|
154
143
|
# 'setter' to add domain entry to the cache in batch (from a file)
|
155
144
|
def file_add(file)
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
164
|
-
end
|
145
|
+
puts "Add entries to the local domains cache table from file: #{file}" if @verbose
|
146
|
+
raise "File non-exist. Please check your file path and name again: #{file}" unless File.exist?(file)
|
147
|
+
changes=Array.new
|
148
|
+
domains=file_2_list(file)
|
149
|
+
changes=bulk_add(domains)
|
150
|
+
rescue => ee
|
151
|
+
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
165
152
|
end
|
166
153
|
|
167
154
|
# 'setter' to add domain entry to the cache in batch (from a list)
|
168
155
|
def bulk_add(list, num=@max_parallel)
|
169
156
|
puts "Add entries to the local domains cache table from list: #{list}" if @verbose
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
results.merge!(process)
|
183
|
-
end
|
157
|
+
results=Hash.new
|
158
|
+
domains=list
|
159
|
+
if domains.size > 0
|
160
|
+
Parallel.map(list, :in_processes => num) { |target|
|
161
|
+
add(target)
|
162
|
+
}.each do |process|
|
163
|
+
if process.nil?
|
164
|
+
next
|
165
|
+
elsif process.empty?
|
166
|
+
#do nothing
|
167
|
+
else
|
168
|
+
results.merge!(process)
|
184
169
|
end
|
185
|
-
@known_internet_domains.merge!(results)
|
186
|
-
puts "Done loading entries."
|
187
|
-
return results
|
188
|
-
else
|
189
|
-
puts "Error: no entry is loaded. Please check your list and try again."
|
190
170
|
end
|
171
|
+
@known_internet_domains.merge!(results)
|
172
|
+
puts "Done loading entries."
|
191
173
|
return results
|
192
|
-
|
193
|
-
puts "
|
174
|
+
else
|
175
|
+
puts "Error: no entry is loaded. Please check your list and try again."
|
194
176
|
end
|
177
|
+
return results
|
178
|
+
rescue => ee
|
179
|
+
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
195
180
|
end
|
196
181
|
alias_method :adds, :bulk_add
|
197
182
|
|
198
183
|
# 'setter' to remove entry from the cache one at a time
|
199
184
|
def delete(domain)
|
200
185
|
puts "Remove entry from the domains cache table: #{domain} " if @verbose
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
puts "Entry not fund. Skipping: #{domain}"
|
209
|
-
end
|
210
|
-
rescue => ee
|
211
|
-
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
186
|
+
domain=domain.strip.downcase
|
187
|
+
if @known_internet_domains.key?(domain)
|
188
|
+
@known_internet_domains.delete(domain)
|
189
|
+
puts "Entry cleared: #{domain}"
|
190
|
+
return domain
|
191
|
+
else
|
192
|
+
puts "Entry not fund. Skipping: #{domain}"
|
212
193
|
end
|
194
|
+
rescue => ee
|
195
|
+
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
213
196
|
end
|
214
197
|
|
215
198
|
# 'setter' to delete domain entry to the cache in batch (from a list)
|
216
199
|
def bulk_delete(list)
|
217
200
|
puts "Delete entries to the local domains cache table from list: #{list}" if @verbose
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
changes.push(domain) unless domain.nil?
|
225
|
-
end
|
226
|
-
puts "Done deleting domains from list: #{list}"
|
227
|
-
return changes
|
228
|
-
else
|
229
|
-
puts "Exception on method bulk_delete: no entry is loaded. Please check your list and try again."
|
201
|
+
domains=list
|
202
|
+
changes=Array.new
|
203
|
+
if domains.size > 0
|
204
|
+
domains.map do |x|
|
205
|
+
domain=delete(x)
|
206
|
+
changes.push(domain) unless domain.nil?
|
230
207
|
end
|
231
|
-
|
232
|
-
|
208
|
+
puts "Done deleting domains from list: #{list}"
|
209
|
+
return changes
|
210
|
+
else
|
211
|
+
puts "Exception on method bulk_delete: no entry is loaded. Please check your list and try again."
|
233
212
|
end
|
213
|
+
rescue => ee
|
214
|
+
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
234
215
|
end
|
235
216
|
alias_method :dels, :bulk_delete
|
236
217
|
|
237
218
|
# 'setter' to delete domain entry to the cache in batch (from a file)
|
238
219
|
def file_delete(file)
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
246
|
-
end
|
220
|
+
puts "Delete entries to the local domains cache table from file: #{file}" if @verbose
|
221
|
+
raise "File non-exist. Please check your file path and name again: #{file}" unless File.exist?(file)
|
222
|
+
domains=file_2_list(file)
|
223
|
+
changes=bulk_delete(domains)
|
224
|
+
rescue => ee
|
225
|
+
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
247
226
|
end
|
248
227
|
|
249
228
|
# 'setter' to remove all entries from the store
|
250
229
|
def delete_all
|
251
230
|
puts "Delete all entries in the domain store! " if @verbose
|
252
|
-
|
253
|
-
|
254
|
-
delete(domain)
|
255
|
-
end
|
256
|
-
rescue => ee
|
257
|
-
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
231
|
+
@known_internet_domains.keys.map do |domain|
|
232
|
+
delete(domain)
|
258
233
|
end
|
234
|
+
rescue => ee
|
235
|
+
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
259
236
|
end
|
260
237
|
|
261
238
|
# Refresh the domain entry one at a time
|
262
239
|
def refresh(domain)
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
puts "Unknown domain: #{domain}"
|
272
|
-
return nil
|
273
|
-
end
|
274
|
-
rescue => ee
|
275
|
-
puts "Exception on method #{__method__} for #{domain}: #{ee}" if @verbose
|
240
|
+
abort "Trusted Internet domain file not loaded properly! " if @known_internet_domains.nil?
|
241
|
+
domain=domain.strip.downcase unless domain.nil?
|
242
|
+
if domain_known?(domain)
|
243
|
+
delete(domain)
|
244
|
+
add(domain)
|
245
|
+
return domain
|
246
|
+
else
|
247
|
+
puts "Unknown domain: #{domain}"
|
276
248
|
return nil
|
277
249
|
end
|
250
|
+
rescue => ee
|
251
|
+
puts "Exception on method #{__method__} for #{domain}: #{ee}" if @verbose
|
252
|
+
return nil
|
278
253
|
end
|
279
254
|
|
280
255
|
# Simple method to check if a domain is already within the domain cache table
|
281
256
|
def domain_known?(domain)
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
return nil
|
292
|
-
end
|
293
|
-
rescue => ee
|
294
|
-
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
257
|
+
#abort "Trusted Internet domain file not loaded properly! " if @known_internet_domains.nil? or @known_internet_sub_domains.nil?
|
258
|
+
domain=domain.strip.downcase unless domain.nil?
|
259
|
+
case self.class.name
|
260
|
+
when "Wmap::DomainTracker"
|
261
|
+
return @known_internet_domains.key?(domain)
|
262
|
+
when "Wmap::DomainTracker::SubDomain"
|
263
|
+
return @known_internet_sub_domains.key?(domain)
|
264
|
+
else
|
265
|
+
return nil
|
295
266
|
end
|
267
|
+
rescue => ee
|
268
|
+
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
296
269
|
return false
|
297
270
|
end
|
298
271
|
alias_method :is_known?, :domain_known?
|
@@ -301,12 +274,10 @@ class Wmap::DomainTracker
|
|
301
274
|
# Dump out the list of known domains
|
302
275
|
def get_domains
|
303
276
|
puts "Retrieve a list of known domain ..." if @verbose
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
return nil
|
309
|
-
end
|
277
|
+
return @known_internet_domains.keys
|
278
|
+
rescue Exception => ee
|
279
|
+
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
280
|
+
return nil
|
310
281
|
end
|
311
282
|
alias_method :dump_domains, :get_domains
|
312
283
|
alias_method :dump, :get_domains
|
@@ -314,19 +285,17 @@ class Wmap::DomainTracker
|
|
314
285
|
# Search potential matching domains from the domain store by using simple regular expression. Note that any upper-case char in the search string will be automatically converted into lower case
|
315
286
|
def search (pattern)
|
316
287
|
puts "Search domain store for the regular expression: #{pattern}" if @verbose
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
results.push(key)
|
323
|
-
end
|
288
|
+
pattern=pattern.strip.downcase
|
289
|
+
results=Array.new
|
290
|
+
@known_internet_domains.keys.map do |key|
|
291
|
+
if key =~ /#{pattern}/i
|
292
|
+
results.push(key)
|
324
293
|
end
|
325
|
-
return results
|
326
|
-
rescue Exception => ee
|
327
|
-
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
328
|
-
return nil
|
329
294
|
end
|
295
|
+
return results
|
296
|
+
rescue Exception => ee
|
297
|
+
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
298
|
+
return nil
|
330
299
|
end
|
331
300
|
alias_method :find, :search
|
332
301
|
|
@@ -340,5 +309,4 @@ class Wmap::DomainTracker
|
|
340
309
|
end
|
341
310
|
alias_method :print, :print_known_domains
|
342
311
|
|
343
|
-
private :load_domains_from_file
|
344
312
|
end
|
@@ -16,19 +16,17 @@ module Wmap
|
|
16
16
|
include Wmap::Utils
|
17
17
|
include Singleton
|
18
18
|
|
19
|
-
attr_accessor :hosts_file, :verbose, :data_dir
|
20
|
-
attr_reader :known_hosts, :known_ips
|
19
|
+
attr_accessor :hosts_file, :verbose, :data_dir, :known_hosts, :known_ips
|
21
20
|
|
22
21
|
# Initialize the instance variables
|
23
22
|
def initialize (params = {})
|
24
23
|
@verbose=params.fetch(:verbose, false)
|
25
24
|
@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../../data/')
|
26
25
|
# Set default instance variables
|
27
|
-
@
|
28
|
-
file=params.fetch(:hosts_file, @file_hosts)
|
26
|
+
@hosts_file=params.fetch(:hosts_file, @data_dir + 'prime_hosts')
|
29
27
|
# Initialize the instance variables
|
30
|
-
File.write(@
|
31
|
-
@known_hosts=load_known_hosts_from_file(
|
28
|
+
File.write(@hosts_file, "") unless File.exist?(@hosts_file)
|
29
|
+
@known_hosts=load_known_hosts_from_file(@hosts_file)
|
32
30
|
@known_ips=Hash.new
|
33
31
|
de_duplicate
|
34
32
|
end
|
@@ -41,7 +39,8 @@ module Wmap
|
|
41
39
|
cns=Hash.new
|
42
40
|
checker=Wmap::UrlChecker.new(:data_dir=>@data_dir)
|
43
41
|
my_tracker = Wmap::SiteTracker.instance
|
44
|
-
my_tracker.
|
42
|
+
my_tracker.sites_file = @data_dir + "sites"
|
43
|
+
my_tracker.load_site_stores_from_file
|
45
44
|
my_tracker.get_ssl_sites.map do |site|
|
46
45
|
puts "Exam SSL enabled site entry #{site} ..."
|
47
46
|
my_host=url_2_host(site)
|
@@ -77,9 +76,10 @@ module Wmap
|
|
77
76
|
puts "Invoke internal procedures to update the primary host-name table from the site store."
|
78
77
|
begin
|
79
78
|
my_tracker=Wmap::SiteTracker.instance
|
80
|
-
my_tracker.
|
79
|
+
my_tracker.sites_file=@data_dir + "sites"
|
80
|
+
my_tracker.load_site_stores_from_file
|
81
81
|
urls = my_tracker.get_redirection_urls
|
82
|
-
my_tracker = nil
|
82
|
+
my_tracker = nil
|
83
83
|
urls.map do |url|
|
84
84
|
if is_url?(url)
|
85
85
|
host=url_2_host(url)
|