wmap 2.5.5 → 2.5.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/{README.rdoc → README.md} +47 -33
- data/bin/wmap +46 -32
- data/lib/wmap/domain_tracker/sub_domain.rb +67 -76
- data/lib/wmap/domain_tracker.rb +176 -208
- data/lib/wmap/host_tracker/primary_host.rb +9 -9
- data/lib/wmap/host_tracker.rb +314 -361
- data/lib/wmap/site_tracker/deactivated_site.rb +3 -4
- data/lib/wmap/site_tracker.rb +586 -640
- data/lib/wmap/utils/url_magic.rb +1 -1
- data/version.txt +2 -2
- data/wmap.gemspec +2 -2
- metadata +4 -5
- data/logs/wmap.log +0 -17
data/lib/wmap/host_tracker.rb
CHANGED
@@ -14,8 +14,8 @@ class Wmap::HostTracker
|
|
14
14
|
include Singleton
|
15
15
|
include Wmap::Utils
|
16
16
|
|
17
|
-
attr_accessor :hosts_file, :max_parallel, :verbose, :data_dir
|
18
|
-
attr_reader :
|
17
|
+
attr_accessor :hosts_file, :max_parallel, :verbose, :data_dir, :known_hosts
|
18
|
+
attr_reader :alias
|
19
19
|
|
20
20
|
# Instance default variables
|
21
21
|
def initialize (params = {})
|
@@ -23,321 +23,298 @@ class Wmap::HostTracker
|
|
23
23
|
@data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../data/')
|
24
24
|
Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir)
|
25
25
|
# Set default instance variables
|
26
|
-
@
|
27
|
-
file=params.fetch(:hosts_file, @file_hosts)
|
26
|
+
@hosts_file=params.fetch(:hosts_file, @data_dir + 'hosts')
|
28
27
|
@max_parallel=params.fetch(:max_parallel, 40)
|
29
28
|
# Initialize the instance variables
|
30
|
-
File.write(@
|
31
|
-
|
29
|
+
File.write(@hosts_file, "") unless File.exist?(@hosts_file)
|
30
|
+
load_known_hosts_from_file(@hosts_file)
|
32
31
|
end
|
33
32
|
|
34
33
|
# Setter to load the known hosts from the local hosts file into a class instance
|
35
|
-
def load_known_hosts_from_file (f_hosts=@
|
36
|
-
#
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
@alias[value]=1
|
57
|
-
end
|
34
|
+
def load_known_hosts_from_file (f_hosts=@hosts_file)
|
35
|
+
puts "Loading local hosts from file: #{f_hosts} ..." if @verbose
|
36
|
+
@known_hosts=Hash.new
|
37
|
+
@alias = Hash.new
|
38
|
+
f=File.open(f_hosts, 'r')
|
39
|
+
f.each do |line|
|
40
|
+
next unless line =~ /\d+\.\d+\.\d+\.\d+/
|
41
|
+
entry=line.chomp.split(%r{\t+|\s+|\,})
|
42
|
+
key=entry[0].downcase
|
43
|
+
value=entry[1]
|
44
|
+
puts "Loading key value pair: #{key} - #{value}" if @verbose
|
45
|
+
@known_hosts[key] = Hash.new unless @known_hosts.key?(key)
|
46
|
+
@known_hosts[key]= value
|
47
|
+
# For reverse host lookup
|
48
|
+
@known_hosts[value] = Hash.new unless @known_hosts.key?(value)
|
49
|
+
@known_hosts[value] = key
|
50
|
+
# Count the number of alias for the recorded IP
|
51
|
+
if @alias.key?(value)
|
52
|
+
@alias[value]+=1
|
53
|
+
else
|
54
|
+
@alias[value]=1
|
58
55
|
end
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
56
|
+
end
|
57
|
+
f.close
|
58
|
+
return @known_hosts
|
59
|
+
#rescue => ee
|
60
|
+
# puts "Exception on method #{__method__}: #{ee}"
|
61
|
+
# return known_hosts
|
65
62
|
end
|
66
63
|
|
67
64
|
# Save the current local hosts hash table into a (random) data repository file
|
68
|
-
def save_known_hosts_to_file!(f_hosts=@
|
69
|
-
#
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
f.write "\n#{key}\t#{@known_hosts[key]}"
|
77
|
-
end
|
65
|
+
def save_known_hosts_to_file!(f_hosts=@hosts_file)
|
66
|
+
puts "Saving the local host repository from memory to file: #{f_hosts} ..."
|
67
|
+
timestamp=Time.now
|
68
|
+
f=File.open(f_hosts, 'w')
|
69
|
+
f.write "# local hosts file created by the #{self.class} class #{__method__} method at: #{timestamp}"
|
70
|
+
@known_hosts.keys.sort.map do |key|
|
71
|
+
unless key =~ /\d+\.\d+\.\d+\.\d+/
|
72
|
+
f.write "\n#{key}\t#{@known_hosts[key]}"
|
78
73
|
end
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
74
|
+
end
|
75
|
+
f.write "\n"
|
76
|
+
f.close
|
77
|
+
puts "local host repository is successfully saved to: #{f_hosts}"
|
78
|
+
#rescue => ee
|
79
|
+
# puts "Exception on method #{__method__}: #{ee}"
|
85
80
|
end
|
86
81
|
alias_method :save!, :save_known_hosts_to_file!
|
87
82
|
|
88
83
|
# Count numbers of entries in the local host repository
|
89
84
|
def count
|
90
85
|
puts "Counting number of entries in the local host repository ..."
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
cnt=cnt+1
|
96
|
-
end
|
86
|
+
cnt=0
|
87
|
+
@known_hosts.keys.map do |key|
|
88
|
+
unless is_ip?(key)
|
89
|
+
cnt=cnt+1
|
97
90
|
end
|
98
|
-
puts "Current number of entries: #{cnt}"
|
99
|
-
return cnt
|
100
|
-
rescue => ee
|
101
|
-
puts "Exception on method #{__method__}: #{ee}"
|
102
91
|
end
|
92
|
+
puts "Current number of entries: #{cnt}"
|
93
|
+
return cnt
|
94
|
+
rescue => ee
|
95
|
+
puts "Exception on method #{__method__}: #{ee}"
|
103
96
|
end
|
104
97
|
|
105
98
|
# Setter to add host entry to the cache once at a time
|
106
99
|
def add(host)
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
tracker
|
100
|
+
puts "Add entry to the local host repository: #{host}"
|
101
|
+
host=host.strip.downcase unless host.nil?
|
102
|
+
unless @known_hosts.key?(host)
|
103
|
+
ip=host_2_ip(host)
|
104
|
+
record=Hash.new
|
105
|
+
if is_ip?(ip)
|
106
|
+
# filter host to known domains only
|
107
|
+
root=get_domain_root(host)
|
108
|
+
puts "Domain root: #{root}" if @verbose
|
109
|
+
domain_tracker=Wmap::DomainTracker.instance
|
110
|
+
domain_tracker.data_dir=@data_dir
|
111
|
+
domain_tracker.domains_file = domain_tracker.data_dir + "domains"
|
112
|
+
domain_tracker.load_domains_from_file
|
113
|
+
if domain_tracker.domain_known?(root)
|
114
|
+
domain_tracker=nil
|
115
|
+
record[host]=ip
|
116
|
+
record[ip]=host
|
117
|
+
puts "Host data repository entry loaded: #{host} <=> #{ip}"
|
118
|
+
# Replace instance with the class variable to avoid potential race condition under parallel engine
|
119
|
+
# add additional logic to update the sub-domain table as well, 02/10/2014
|
120
|
+
sub=get_sub_domain(host)
|
121
|
+
if sub!=root
|
122
|
+
tracker=Wmap::DomainTracker::SubDomain.instance
|
123
|
+
tracker.data_dir=@data_dir
|
124
|
+
tracker.sub_domains_file = tracker.data_dir + "sub_domains"
|
125
|
+
tracker.known_internet_sub_domains=tracker.load_domains_from_file(tracker.sub_domains_file)
|
126
|
+
unless tracker.domain_known?(sub)
|
127
|
+
tracker.add(sub)
|
128
|
+
tracker.save!
|
135
129
|
end
|
136
|
-
|
137
|
-
return record
|
138
|
-
else
|
139
|
-
domain_tracker=nil
|
140
|
-
puts "Error - host #{host} has an untrusted internet root domain: #{root}\nPlease update the trusted domain seeds file first if necessary."
|
130
|
+
tracker=nil
|
141
131
|
end
|
132
|
+
@known_hosts.merge!(record)
|
133
|
+
return record
|
142
134
|
else
|
143
|
-
|
135
|
+
domain_tracker=nil
|
136
|
+
puts "Error - host #{host} has an untrusted internet root domain: #{root}\nPlease update the trusted domain seeds file first if necessary."
|
144
137
|
end
|
145
138
|
else
|
146
|
-
puts "
|
139
|
+
puts "Problem resolve host #{host} - unknown IP: #{ip}"
|
147
140
|
end
|
148
|
-
|
149
|
-
puts "
|
141
|
+
else
|
142
|
+
puts "Host is already exist. Skip: #{host}"
|
150
143
|
end
|
144
|
+
rescue => ee
|
145
|
+
puts "Exception on method #{__method__}: #{ee}" if @verbose
|
151
146
|
end
|
152
147
|
|
153
148
|
# Setter to add host entry to the local hosts in batch (from an array)
|
154
149
|
def bulk_add(list, num=@max_parallel)
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
results.merge!(process)
|
169
|
-
end
|
150
|
+
puts "Add entries to the local host repository: #{list}"
|
151
|
+
results=Hash.new
|
152
|
+
if list.size > 0
|
153
|
+
puts "Start parallel host update processing on:\n #{list}" if @verbose
|
154
|
+
Parallel.map(list, :in_processes => num) { |target|
|
155
|
+
add(target)
|
156
|
+
}.each do |process|
|
157
|
+
if process.nil?
|
158
|
+
next
|
159
|
+
elsif process.empty?
|
160
|
+
#do nothing
|
161
|
+
else
|
162
|
+
results.merge!(process)
|
170
163
|
end
|
171
|
-
@known_hosts.merge!(results)
|
172
|
-
puts "Done loading entries."
|
173
|
-
return results
|
174
|
-
else
|
175
|
-
puts "Error: empty list - no entry is loaded. Please check your input list and try again."
|
176
164
|
end
|
165
|
+
@known_hosts.merge!(results)
|
166
|
+
puts "Done loading entries."
|
177
167
|
return results
|
178
|
-
|
179
|
-
puts "
|
168
|
+
else
|
169
|
+
puts "Error: empty list - no entry is loaded. Please check your input list and try again."
|
180
170
|
end
|
171
|
+
return results
|
172
|
+
rescue => ee
|
173
|
+
puts "Exception on method #{__method__}: #{ee}"
|
181
174
|
end
|
182
175
|
alias_method :adds, :bulk_add
|
183
176
|
|
184
177
|
# 'setter' to add host entry to the local hosts in batch (from a file)
|
185
178
|
def file_add(file)
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
puts "Exception on method #{__method__}: #{ee}"
|
194
|
-
end
|
179
|
+
puts "Add entries to the local host repository from file: #{file}"
|
180
|
+
raise "File non-exist. Please check your file path and name again: #{file}" unless File.exist?(file)
|
181
|
+
hosts=file_2_list(file)
|
182
|
+
changes=bulk_add(hosts)
|
183
|
+
return changes
|
184
|
+
rescue => ee
|
185
|
+
puts "Exception on method #{__method__}: #{ee}"
|
195
186
|
end
|
196
187
|
|
197
188
|
# 'setter' to remove entry from the local hosts one at a time
|
198
189
|
def delete(host)
|
199
190
|
puts "Remove entry from the local host repository: #{host} "
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
puts "Entry not fund. Skip: #{host}"
|
208
|
-
end
|
209
|
-
rescue => ee
|
210
|
-
puts "Exception on method #{__method__}: #{ee}"
|
191
|
+
host=host.strip.downcase
|
192
|
+
if @known_hosts.key?(host)
|
193
|
+
@known_hosts.delete(host)
|
194
|
+
puts "Entry cleared."
|
195
|
+
return host
|
196
|
+
else
|
197
|
+
puts "Entry not fund. Skip: #{host}"
|
211
198
|
end
|
199
|
+
rescue => ee
|
200
|
+
puts "Exception on method #{__method__}: #{ee}"
|
212
201
|
end
|
213
202
|
|
214
203
|
# 'setter' to delete host entry to the cache in batch (from an array)
|
215
204
|
def bulk_delete(list)
|
216
205
|
puts "Delete entries to the local host repository from:\n #{list}"
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
changes.push(host) unless host.nil?
|
224
|
-
end
|
225
|
-
puts "Done deleting hosts."
|
226
|
-
return changes
|
227
|
-
else
|
228
|
-
puts "Error: empty list - no entry is loaded. Please check your list and try again."
|
206
|
+
hosts=list
|
207
|
+
changes=Array.new
|
208
|
+
if hosts.size > 0
|
209
|
+
hosts.map do |x|
|
210
|
+
host=delete(x)
|
211
|
+
changes.push(host) unless host.nil?
|
229
212
|
end
|
230
|
-
|
231
|
-
|
213
|
+
puts "Done deleting hosts."
|
214
|
+
return changes
|
215
|
+
else
|
216
|
+
puts "Error: empty list - no entry is loaded. Please check your list and try again."
|
232
217
|
end
|
218
|
+
rescue => ee
|
219
|
+
puts "Exception on method #{__method__}: #{ee}"
|
233
220
|
end
|
234
221
|
alias_method :dels, :bulk_delete
|
235
222
|
|
236
223
|
# Setter to delete host entries in the local hosts in batch (from a file)
|
237
224
|
def file_delete(file)
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
puts "Exception on method #{__method__}: #{ee}"
|
247
|
-
end
|
225
|
+
puts "Delete the local host repository entries from file: #{file}"
|
226
|
+
raise "File non-exist. Please check your file path and name again: #{file}" unless File.exist?(file)
|
227
|
+
hosts=file_2_list(file)
|
228
|
+
changes=bulk_delete(hosts)
|
229
|
+
puts "Delete done."
|
230
|
+
return changes
|
231
|
+
rescue => ee
|
232
|
+
puts "Exception on method #{__method__}: #{ee}"
|
248
233
|
end
|
249
234
|
|
250
235
|
# Setter to refresh the entry from the cache one at a time
|
251
236
|
def refresh(host)
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
if
|
259
|
-
|
260
|
-
|
261
|
-
return nil
|
262
|
-
else
|
263
|
-
@known_hosts[host]=new_ip
|
264
|
-
@known_hosts[new_ip]=host
|
265
|
-
puts "Entry refreshed: #{host}\t#{@known_hosts[host]}"
|
266
|
-
return host
|
267
|
-
end
|
237
|
+
puts "Refresh the local host repository for host: #{host} "
|
238
|
+
host=host.strip.downcase
|
239
|
+
if @known_hosts.key?(host)
|
240
|
+
old_ip=@known_hosts[host]
|
241
|
+
new_ip=host_2_ip(host)
|
242
|
+
if is_ip?(new_ip)
|
243
|
+
if old_ip==new_ip
|
244
|
+
puts "No change for the host entry: #{host}\t#{old_ip}"
|
245
|
+
return nil
|
268
246
|
else
|
269
|
-
|
270
|
-
@known_hosts
|
247
|
+
@known_hosts[host]=new_ip
|
248
|
+
@known_hosts[new_ip]=host
|
249
|
+
puts "Entry refreshed: #{host}\t#{@known_hosts[host]}"
|
271
250
|
return host
|
272
251
|
end
|
273
252
|
else
|
274
|
-
puts "
|
253
|
+
puts "Host can no longer be resolved in the Internet. Entry removed: #{host}\t#{@known_hosts[host]}"
|
254
|
+
@known_hosts.delete(host)
|
255
|
+
return host
|
275
256
|
end
|
276
|
-
|
277
|
-
puts "
|
257
|
+
else
|
258
|
+
puts "Error entry non exist: #{host}"
|
278
259
|
end
|
260
|
+
rescue => ee
|
261
|
+
puts "Exception on method #{__method__}: #{ee}"
|
279
262
|
end
|
280
263
|
|
281
264
|
# Refresh all the entries in the local hosts by querying the Internet
|
282
265
|
def refresh_all
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
puts "Exception on method #{__method__}: #{ee}"
|
302
|
-
end
|
266
|
+
puts "Refresh all the entries in the local host repository in one shot."
|
267
|
+
changes=Hash.new
|
268
|
+
hosts=@known_hosts.keys
|
269
|
+
@known_hosts=Hash.new
|
270
|
+
changes=bulk_add(hosts)
|
271
|
+
@known_hosts.merge!(changes)
|
272
|
+
#@known_hosts.keys.map do |key|
|
273
|
+
# unless is_ip?(key)
|
274
|
+
# host=refresh(key)
|
275
|
+
# changes.push(host) unless host.nil?
|
276
|
+
# end
|
277
|
+
#end
|
278
|
+
puts "\n#{changes.size} Entries Refreshed:" if changes.size>0
|
279
|
+
#changes.map { |x| puts x }
|
280
|
+
puts "Done refreshing the local hosts."
|
281
|
+
return changes
|
282
|
+
rescue => ee
|
283
|
+
puts "Exception on method #{__method__}: #{ee}"
|
303
284
|
end
|
304
285
|
|
305
286
|
# Extract known root domains from the local host repository @known_hosts
|
306
287
|
def get_root_domains
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
zones.push(zone) unless zone.nil?
|
315
|
-
end
|
316
|
-
zones.uniq!.sort!
|
317
|
-
return zones
|
318
|
-
rescue => ee
|
319
|
-
puts "Exception on method #{__method__}: #{ee}"
|
288
|
+
puts "Dump out all active root domains from the cache."
|
289
|
+
zones=Array.new
|
290
|
+
(@known_hosts.keys-["",nil]).map do |hostname|
|
291
|
+
next if is_ip?(hostname)
|
292
|
+
hostname = hostname.strip
|
293
|
+
zone = get_domain_root(hostname)
|
294
|
+
zones.push(zone) unless zone.nil?
|
320
295
|
end
|
296
|
+
zones.uniq!.sort!
|
297
|
+
return zones
|
298
|
+
rescue => ee
|
299
|
+
puts "Exception on method #{__method__}: #{ee}"
|
321
300
|
end
|
322
301
|
alias_method :dump_root_domains, :get_root_domains
|
323
302
|
|
324
303
|
# Extract hostname without the root domain part from the @known_hosts. Data can be used for statistics study.
|
325
304
|
def get_a_records
|
326
305
|
puts "Dump out all known A records from the local hosts."
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
records.push(record) unless record.nil?
|
335
|
-
end
|
336
|
-
records.sort!
|
337
|
-
return records
|
338
|
-
rescue => ee
|
339
|
-
puts "Exception on method #{__method__}: #{ee}"
|
306
|
+
records=Array.new
|
307
|
+
(@known_hosts.keys-["",nil]).map do |hostname|
|
308
|
+
next if is_ip?(hostname)
|
309
|
+
hostname = hostname.strip
|
310
|
+
root = get_domain_root(hostname)
|
311
|
+
record = hostname.sub('.'+root,'')
|
312
|
+
records.push(record) unless record.nil?
|
340
313
|
end
|
314
|
+
records.sort!
|
315
|
+
return records
|
316
|
+
rescue => ee
|
317
|
+
puts "Exception on method #{__method__}: #{ee}"
|
341
318
|
end
|
342
319
|
alias_method :dump_a_records, :get_a_records
|
343
320
|
|
@@ -356,160 +333,139 @@ class Wmap::HostTracker
|
|
356
333
|
# Print summary report on the cache
|
357
334
|
def print_host(host)
|
358
335
|
puts "Local host store entry for #{host}"
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
puts "Unknown host in the local store: #{host}"
|
367
|
-
end
|
368
|
-
rescue => ee
|
369
|
-
puts "Exception on method #{__method__}: #{ee}"
|
336
|
+
host.strip!
|
337
|
+
raise "Invalid input: #{host}" unless is_fqdn?(host)
|
338
|
+
if @known_hosts.key?(host)
|
339
|
+
value=@known_hosts[host]
|
340
|
+
puts "#{host}\t#{value}"
|
341
|
+
else
|
342
|
+
puts "Unknown host in the local store: #{host}"
|
370
343
|
end
|
344
|
+
rescue => ee
|
345
|
+
puts "Exception on method #{__method__}: #{ee}"
|
371
346
|
end
|
372
347
|
alias_method :print, :print_host
|
373
348
|
|
374
349
|
# Check if the specific IP within @known_hosts table
|
375
350
|
def ip_known? (ip)
|
376
351
|
known = false
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
puts "IP Lookup Error: #{ee}"
|
384
|
-
end
|
385
|
-
return false
|
386
|
-
end
|
387
|
-
return known
|
352
|
+
ip=ip.strip unless ip.nil?
|
353
|
+
return false if @known_hosts==nil
|
354
|
+
return @known_hosts.key?(ip.strip)
|
355
|
+
rescue => ee
|
356
|
+
puts "IP Lookup Error: #{ee}" if @verbose
|
357
|
+
return false
|
388
358
|
end
|
389
359
|
alias_method :has_a_record?, :ip_known?
|
390
360
|
|
391
361
|
# Check if the specific host within @known_hosts table
|
392
362
|
def host_known? (host)
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
puts "Host Lookup Error: #{ee}"
|
400
|
-
end
|
401
|
-
return false
|
363
|
+
host=host.strip.downcase unless host.nil?
|
364
|
+
return false if @known_hosts==nil
|
365
|
+
return @known_hosts.key?(host.strip)
|
366
|
+
rescue => ee
|
367
|
+
if @verbose
|
368
|
+
puts "Host Lookup Error: #{ee}"
|
402
369
|
end
|
370
|
+
return false
|
403
371
|
end
|
404
372
|
alias_method :is_known?, :host_known?
|
405
373
|
|
406
374
|
# Perform reverse DNS lookup on the local host repository. Not to confuse with the reverse DNS lookup from the Internet
|
407
375
|
def local_ip_2_host (ip)
|
408
376
|
puts "Reverse DNS lookup from the local host repository" if @verbose
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
return nil
|
415
|
-
end
|
416
|
-
rescue => ee
|
417
|
-
puts "Exception on method #{__method__}: #{ee}"
|
377
|
+
ip=ip.strip unless ip.nil?
|
378
|
+
if @known_hosts.key?(ip)
|
379
|
+
return @known_hosts[ip]
|
380
|
+
else
|
381
|
+
return nil
|
418
382
|
end
|
383
|
+
rescue => ee
|
384
|
+
puts "Exception on method #{__method__}: #{ee}"
|
419
385
|
return nil
|
420
386
|
end
|
421
387
|
|
422
388
|
# Perform DNS lookup on the local host repository. Not to confuse with the DNS lookup from the Internet
|
423
389
|
def local_host_2_ip (host)
|
424
390
|
puts "DNS lookup from the local host repository" if @verbose
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
else
|
430
|
-
return nil
|
431
|
-
end
|
432
|
-
rescue => ee
|
433
|
-
puts "Exception on method #{__method__}: #{ee}"
|
391
|
+
host=host.strip unless host.nil?
|
392
|
+
if @known_hosts.key?(host)
|
393
|
+
return @known_hosts[host]
|
394
|
+
else
|
434
395
|
return nil
|
435
396
|
end
|
397
|
+
rescue => ee
|
398
|
+
puts "Exception on method #{__method__}: #{ee}"
|
399
|
+
return nil
|
436
400
|
end
|
437
401
|
|
438
402
|
# Extract a list of sub-domains from the local host repository @known_hosts
|
439
403
|
def dump_sub_domains
|
440
404
|
puts "Dump out all active sub domains from the local hosts." if @verbose
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
subs.push(sub) unless sub.nil?
|
448
|
-
end
|
449
|
-
subs.uniq!.sort! unless subs.empty?
|
450
|
-
puts "Found sub domains: #{subs}" if @verbose
|
451
|
-
return subs
|
452
|
-
rescue Exception => ee
|
453
|
-
puts "Exception on method #{__method__}: #{ee}"
|
454
|
-
return subs
|
405
|
+
subs=Array.new
|
406
|
+
@known_hosts.keys.each do |hostname|
|
407
|
+
next if is_ip?(hostname)
|
408
|
+
hostname = hostname.strip
|
409
|
+
sub = get_subdomain(hostname)
|
410
|
+
subs.push(sub) unless sub.nil?
|
455
411
|
end
|
412
|
+
subs.uniq!.sort! unless subs.empty?
|
413
|
+
puts "Found sub domains: #{subs}" if @verbose
|
414
|
+
return subs
|
415
|
+
rescue Exception => ee
|
416
|
+
puts "Exception on method #{__method__}: #{ee}"
|
417
|
+
return subs
|
456
418
|
end
|
457
419
|
alias_method :get_sub_domains, :dump_sub_domains
|
458
420
|
|
459
421
|
# Based on the current host store, to determine if an entry is a known sub-domain
|
460
422
|
def sub_domain_known?(domain)
|
461
423
|
puts "Validate sub-domain: #{domain}" if @verbose
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
puts "Exception on method #{__method__}: #{ee}"
|
468
|
-
end
|
424
|
+
domain=domain.strip.downcase
|
425
|
+
subs=dump_sub_domains
|
426
|
+
return subs.include?(domain)
|
427
|
+
rescue Exception => ee
|
428
|
+
puts "Exception on method #{__method__}: #{ee}"
|
469
429
|
end
|
470
430
|
|
471
431
|
# Search potential matching sites from the host store by using simple regular expression. Note that any upper-case char in the search string will be automatically converted into lower case
|
472
432
|
def search (pattern)
|
473
433
|
puts "Search host store based on the regular expression: #{pattern}" if @verbose
|
474
|
-
|
475
|
-
|
476
|
-
|
477
|
-
|
478
|
-
|
479
|
-
results.push(key)
|
480
|
-
end
|
434
|
+
pattern=pattern.strip.downcase
|
435
|
+
results=Array.new
|
436
|
+
@known_hosts.keys.map do |key|
|
437
|
+
if key =~ /#{pattern}/i
|
438
|
+
results.push(key)
|
481
439
|
end
|
482
|
-
return results
|
483
|
-
rescue Exception => ee
|
484
|
-
puts "Exception on method #{__method__}: #{ee}"
|
485
|
-
return nil
|
486
440
|
end
|
441
|
+
return results
|
442
|
+
rescue Exception => ee
|
443
|
+
puts "Exception on method #{__method__}: #{ee}"
|
444
|
+
return nil
|
487
445
|
end
|
488
446
|
alias_method :find, :search
|
489
447
|
|
490
448
|
# Search local host repository and return a list of aliases for the host
|
491
449
|
def host_aliases (host)
|
492
450
|
puts "Search aliases in the local hosts data repository for host: #{host}" if @verbose
|
493
|
-
|
494
|
-
|
495
|
-
|
496
|
-
|
497
|
-
|
498
|
-
|
499
|
-
|
500
|
-
|
501
|
-
|
502
|
-
aliases.push(key)
|
503
|
-
end
|
451
|
+
host.strip!
|
452
|
+
raise "Unknown method input: #{host} We expect a FQDN host-name string from you. " unless is_fqdn?(host)
|
453
|
+
aliases=Array.new
|
454
|
+
if @known_hosts.key?(host)
|
455
|
+
ip=local_host_2_ip(host)
|
456
|
+
@known_hosts.keys.map do |key|
|
457
|
+
my_ip=local_host_2_ip(key)
|
458
|
+
if ip == my_ip
|
459
|
+
aliases.push(key)
|
504
460
|
end
|
505
|
-
else
|
506
|
-
raise "Unknown host-name in the local hosts data repository: #{host}"
|
507
461
|
end
|
508
|
-
|
509
|
-
|
510
|
-
puts "Exception on method #{__method__}: #{ee}"
|
511
|
-
return nil
|
462
|
+
else
|
463
|
+
raise "Unknown host-name in the local hosts data repository: #{host}"
|
512
464
|
end
|
465
|
+
return aliases-[host]
|
466
|
+
rescue Exception => ee
|
467
|
+
puts "Exception on method #{__method__}: #{ee}"
|
468
|
+
return nil
|
513
469
|
end
|
514
470
|
alias_method :aliases, :host_aliases
|
515
471
|
|
@@ -519,40 +475,37 @@ class Wmap::HostTracker
|
|
519
475
|
h=Hash.new
|
520
476
|
host_store=Hash.new
|
521
477
|
top=Array.new
|
522
|
-
|
523
|
-
|
524
|
-
|
525
|
-
|
526
|
-
|
527
|
-
|
528
|
-
|
529
|
-
|
530
|
-
|
531
|
-
|
532
|
-
|
533
|
-
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
|
538
|
-
|
539
|
-
|
540
|
-
|
541
|
-
|
542
|
-
h[host[0]]=1
|
543
|
-
end
|
544
|
-
end
|
545
|
-
result = h.keys.sort { |a,b| h[b] <=> h[a] } # Sort by value descendantly
|
546
|
-
num = result.size if result.size < num
|
547
|
-
for i in 0...num
|
548
|
-
top.push(result[i])
|
478
|
+
# Build a host table from the host file
|
479
|
+
f=File.open(@hosts_file, 'r')
|
480
|
+
f.each do |line|
|
481
|
+
next unless line =~ /\d+\.\d+\.\d+\.\d+/
|
482
|
+
# skip the domain roots in the host list
|
483
|
+
next if is_domain_root?(line.chomp)
|
484
|
+
entry=line.chomp.split(%r{\t+|\s+|\,})
|
485
|
+
key=entry[0].downcase
|
486
|
+
value=entry[1]
|
487
|
+
puts "Loading key value pair: #{key} - #{value}" if @verbose
|
488
|
+
host_store[key] = Hash.new unless known_hosts.key?(key)
|
489
|
+
host_store[key]= value
|
490
|
+
end
|
491
|
+
f.close
|
492
|
+
host_store.keys.map do |key|
|
493
|
+
host=key.split('.')
|
494
|
+
if h.key?(host[0])
|
495
|
+
h[host[0]]+=1
|
496
|
+
else
|
497
|
+
h[host[0]]=1
|
549
498
|
end
|
550
|
-
return top
|
551
|
-
rescue Exception => ee
|
552
|
-
puts "Exception on method #{__method__}: #{ee}"
|
553
|
-
return nil
|
554
499
|
end
|
500
|
+
result = h.keys.sort { |a,b| h[b] <=> h[a] } # Sort by value descendantly
|
501
|
+
num = result.size if result.size < num
|
502
|
+
for i in 0...num
|
503
|
+
top.push(result[i])
|
504
|
+
end
|
505
|
+
return top
|
506
|
+
rescue Exception => ee
|
507
|
+
puts "Exception on method #{__method__}: #{ee}"
|
508
|
+
return nil
|
555
509
|
end
|
556
510
|
|
557
|
-
private :load_known_hosts_from_file
|
558
511
|
end
|