icaprb-filter 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.rspec +2 -0
- data/.travis.yml +4 -0
- data/Gemfile +4 -0
- data/LICENSE +25 -0
- data/README.md +68 -0
- data/README.rdoc +22 -0
- data/Rakefile +12 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/bin/start_server.rb +40 -0
- data/icaprb-filter.gemspec +33 -0
- data/lib/icaprb/filter.rb +189 -0
- data/lib/icaprb/filter/service.rb +107 -0
- data/lib/icaprb/filter/solution.rb +954 -0
- data/lib/icaprb/filter/steganography.rb +91 -0
- data/lib/icaprb/filter/version.rb +8 -0
- data/samples/block_samples/check_hashes_test.rb +7 -0
- data/samples/block_samples/check_hashes_test.txt +2 -0
- data/samples/block_samples/content_has_key_test.html +10 -0
- data/samples/block_samples/content_has_key_test.rb +6 -0
- data/samples/block_samples/content_includes_test.html +10 -0
- data/samples/block_samples/content_includes_test.rb +6 -0
- data/samples/block_samples/nothing_to_filter_test.html +10 -0
- data/samples/block_samples/url_contains_test.rb +6 -0
- data/samples/custom_samples/virustotal_config.rb +6 -0
- metadata +205 -0
@@ -0,0 +1,107 @@
|
|
1
|
+
require 'icaprb/server'
|
2
|
+
require 'icaprb/filter'
|
3
|
+
|
4
|
+
# nodoc
|
5
|
+
module ICAPrb
|
6
|
+
# Server
|
7
|
+
module Server
|
8
|
+
# The filter service, reference in your startup script
|
9
|
+
class FilterService < ICAPrb::Server::Services::ServiceBase
|
10
|
+
# Load configuration for filter service
|
11
|
+
# +path+:: Path to the configuration file
|
12
|
+
def initialize(path, timeout=nil)
|
13
|
+
super('filter',[:request_mod, :response_mod],nil,60,nil,nil,nil,1000)
|
14
|
+
@timeout = timeout
|
15
|
+
@request_list, @response_list = ::ICAPrb::Filter.load_filters(path)
|
16
|
+
end
|
17
|
+
# Process a new filter request
|
18
|
+
# +icap_server+: ICAP server data
|
19
|
+
# +ip+:: IP
|
20
|
+
# +socket+:: Socket
|
21
|
+
# +data+:: Data for the current request
|
22
|
+
def process_request(icap_server,ip,socket,data)
|
23
|
+
logger = icap_server.logger
|
24
|
+
ICAPrb::Filter.set_logger(logger)
|
25
|
+
ICAPrb::Filters.set_logger(logger)
|
26
|
+
logger.debug '==========================================='
|
27
|
+
logger.debug 'Start processing data via filter service...'
|
28
|
+
logger.debug 'Request url: ' + data[:http_request_header]["Host"].to_s
|
29
|
+
|
30
|
+
#logger.debug data
|
31
|
+
# Go through all filters for resp or req
|
32
|
+
begin
|
33
|
+
response = ::ICAPrb::Server::Response.new
|
34
|
+
is_modified = false
|
35
|
+
if data[:icap_data][:request_line][:icap_method] == :request_mod
|
36
|
+
logger.debug('Event is REQUEST')
|
37
|
+
@request_list.each do |entry|
|
38
|
+
logger.debug('Executing filter: ' + entry[:name].to_s)
|
39
|
+
filter_return_value = entry[:object].plugin(data)
|
40
|
+
if [true, false].include? filter_return_value
|
41
|
+
is_modified = (filter_return_value || is_modified)
|
42
|
+
logger.debug("Filter return value: " + filter_return_value.to_s)
|
43
|
+
else
|
44
|
+
logger.error("Filter " + entry[:name] + ' has illegal return value')
|
45
|
+
end
|
46
|
+
end
|
47
|
+
elsif data[:icap_data][:request_line][:icap_method] == :response_mod
|
48
|
+
logger.debug('Event is RESPONSE')
|
49
|
+
@response_list.each do |entry|
|
50
|
+
logger.debug('Executing plugin: ' + entry[:name].to_s)
|
51
|
+
plugin_return_value = entry[:object].plugin(data)
|
52
|
+
if [true, false].include? plugin_return_value
|
53
|
+
is_modified = (plugin_return_value || is_modified)
|
54
|
+
logger.debug("Plugin return value: " + plugin_return_value.to_s)
|
55
|
+
else
|
56
|
+
logger.error("Plugin " + entry[:name] + ' has illegal return value')
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
if is_modified
|
61
|
+
logger.debug('Data was modified, sending everything')
|
62
|
+
else
|
63
|
+
logger.debug('Data is unmodified, sending status code 204')
|
64
|
+
end
|
65
|
+
# Nothing changed
|
66
|
+
unless is_modified
|
67
|
+
# Send 204 (unmodified) flag, no content required
|
68
|
+
response.icap_status_code = 204
|
69
|
+
response.write_headers_to_socket socket
|
70
|
+
return
|
71
|
+
end
|
72
|
+
# Something changed
|
73
|
+
if data[:icap_data][:request_line][:icap_method] == :response_mod
|
74
|
+
http_header = data[:http_response_header]
|
75
|
+
if data[:http_response_body]
|
76
|
+
http_body = ICAPrb::Server::ResponseBody.new(data[:http_response_body])
|
77
|
+
http_header['Content-Length'] = http_body.length
|
78
|
+
end
|
79
|
+
else
|
80
|
+
http_header = data[:http_request_header]
|
81
|
+
if data[:http_request_body]
|
82
|
+
http_body = ICAPrb::Server::RequestBody.new(data[:http_request_body])
|
83
|
+
http_header['Content-Length'] = http_body.length
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
response.components << http_header
|
88
|
+
response.components << http_body if http_body
|
89
|
+
response.icap_status_code = 200
|
90
|
+
response.write_headers_to_socket socket
|
91
|
+
socket.write(http_body.to_chunk)
|
92
|
+
::ICAPrb::Server::Response.send_last_chunk(socket,false)
|
93
|
+
rescue StandardError => error
|
94
|
+
logger.warn 'Error in filter framework'
|
95
|
+
logger.warn error.message
|
96
|
+
logger.warn error.backtrace
|
97
|
+
logger.warn error.backtrace_locations
|
98
|
+
# Don't anger the consumer
|
99
|
+
response = ::ICAPrb::Server::Response.new
|
100
|
+
response.icap_status_code = 204
|
101
|
+
response.write_headers_to_socket socket
|
102
|
+
end
|
103
|
+
logger.debug '==========================================='
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
@@ -0,0 +1,954 @@
|
|
1
|
+
require 'pdf/reader'
|
2
|
+
require 'nokogiri'
|
3
|
+
require 'icaprb/server'
|
4
|
+
require 'rest-client'
|
5
|
+
require 'json'
|
6
|
+
require 'digest'
|
7
|
+
require 'uirusu'
|
8
|
+
|
9
|
+
# ICAPrb
|
10
|
+
module ICAPrb
|
11
|
+
# Methods for filters
|
12
|
+
class Filters
|
13
|
+
# Set logger for filter
|
14
|
+
# +logger+:: Logger object
|
15
|
+
def self.set_logger(logger)
|
16
|
+
@logger = logger
|
17
|
+
end
|
18
|
+
|
19
|
+
# Get logger
|
20
|
+
def self.get_logger
|
21
|
+
@logger
|
22
|
+
end
|
23
|
+
|
24
|
+
# Generate class reference from string
|
25
|
+
# +str+:: String to get the class from
|
26
|
+
def self.class_from_string(str)
|
27
|
+
str.split('::').inject(Object) do |mod, class_name|
|
28
|
+
mod.const_get(class_name)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
# Get all available filters in the namespace ICAPrb::FilterSolutions::
|
32
|
+
def self.get_filters
|
33
|
+
filter_symbols = FilterSolutions.constants.select{ |c| FilterSolutions.const_get(c).is_a? Class }
|
34
|
+
filters = []
|
35
|
+
filter_symbols.each { |filter_symbol|
|
36
|
+
symbol_path = 'ICAPrb::FilterSolutions::' + filter_symbol.to_s
|
37
|
+
current_class = self.class_from_string(symbol_path)
|
38
|
+
filters << {:class => current_class, :name => current_class::FILTER_NAME, :modes => current_class::MODES}
|
39
|
+
}
|
40
|
+
filters
|
41
|
+
end
|
42
|
+
|
43
|
+
# Get all available plugins in the namespace ICAPrb::Plugins::
|
44
|
+
def self.get_plugins
|
45
|
+
plugin_symbols = Plugins.constants.select{ |c| Plugins.const_get(c).is_a? Class }
|
46
|
+
plugins = []
|
47
|
+
plugin_symbols.each { |plugin_symbol|
|
48
|
+
symbol_path = 'ICAPrb::Plugins::' + plugin_symbol.to_s
|
49
|
+
current_class = self.class_from_string(symbol_path)
|
50
|
+
plugins << {:class => current_class, :name => current_class::PLUGIN_NAME, :modes =>current_class::MODES}
|
51
|
+
}
|
52
|
+
plugins
|
53
|
+
end
|
54
|
+
end
|
55
|
+
# Contains block rule definitions
|
56
|
+
module FilterSolutions
|
57
|
+
# Check for hashes and block if anything matches
|
58
|
+
class HashedContent
|
59
|
+
# Name in the configuration file
|
60
|
+
FILTER_NAME = 'check_hashes'
|
61
|
+
# Available mod modes
|
62
|
+
MODES = [:response_mod]
|
63
|
+
# Constructor
|
64
|
+
# +mode+:: resp or req mod
|
65
|
+
# +parameters+:: All parameters given in the configuration file
|
66
|
+
def initialize(_, parameters)
|
67
|
+
@params = []
|
68
|
+
file_path = parameters[0]
|
69
|
+
File.foreach(File.expand_path(file_path)) do |line|
|
70
|
+
line = line.chomp.gsub('\n', '').gsub('\r', '')
|
71
|
+
@params << line
|
72
|
+
end
|
73
|
+
end
|
74
|
+
# Execute plugin
|
75
|
+
# +data+:: ICAP data
|
76
|
+
def plugin(data)
|
77
|
+
if data[:http_response_body] == nil
|
78
|
+
ICAPrb::Filters.get_logger.debug('Nothing to hash')
|
79
|
+
return false
|
80
|
+
end
|
81
|
+
hash = Digest::SHA256.hexdigest(data[:http_response_body]).to_s
|
82
|
+
ICAPrb::Filters.get_logger.debug('Calculated hash: ' + hash.to_s)
|
83
|
+
@params.each do |other_hash|
|
84
|
+
if hash.eql? other_hash
|
85
|
+
# Block by telling the user that this is blocked
|
86
|
+
ICAPrb::Filters.get_logger.debug('Found match')
|
87
|
+
data[:http_response_body] = '<html><body>Blocked Hash</body></html>'
|
88
|
+
return true
|
89
|
+
end
|
90
|
+
end
|
91
|
+
false
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
# Check if the URL contains the given words
|
96
|
+
class URLContains
|
97
|
+
# Name in the configuration file
|
98
|
+
FILTER_NAME = 'url_contains'
|
99
|
+
# Available mod modes
|
100
|
+
MODES = [:response_mod]
|
101
|
+
|
102
|
+
# Constructor
|
103
|
+
# +mode+:: resp or req mod
|
104
|
+
# +parameters+:: All parameters given in the configuration file
|
105
|
+
def initialize(_, parameters)
|
106
|
+
@params = parameters
|
107
|
+
end
|
108
|
+
|
109
|
+
# Execute plugin
|
110
|
+
# +data+:: ICAP data
|
111
|
+
def plugin(data)
|
112
|
+
uri = data[:http_request_header]['Host']
|
113
|
+
@params.each { |url|
|
114
|
+
if uri.include? url
|
115
|
+
# Block by telling the user that this is blocked
|
116
|
+
data[:http_response_body] = '<html><body>Blocked URI</body></html>'
|
117
|
+
return true
|
118
|
+
end
|
119
|
+
}
|
120
|
+
false
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
# Check in headers for regex
|
125
|
+
class Headers
|
126
|
+
# Name in the configuration file
|
127
|
+
FILTER_NAME = 'headers'
|
128
|
+
# Available mod modes
|
129
|
+
MODES = [:response_mod]
|
130
|
+
# Constructor
|
131
|
+
# +mode+:: resp or req mod
|
132
|
+
# +parameters+:: All parameters given in the configuration file
|
133
|
+
def initialize(_, parameters)
|
134
|
+
@params = parameters
|
135
|
+
end
|
136
|
+
# Execute plugin
|
137
|
+
# +data+:: ICAP data
|
138
|
+
def plugin(data)
|
139
|
+
headers = data[:http_request_header]
|
140
|
+
headers.each { |header_name, header_content|
|
141
|
+
@params.each { |header_target, regex_data|
|
142
|
+
if header_name == header_target
|
143
|
+
regex_data.each{ |current_regex|
|
144
|
+
if header_content.match(current_regex)
|
145
|
+
data[:http_response_body] = '<html><body>Blocked header</body></html>'
|
146
|
+
return true
|
147
|
+
end
|
148
|
+
}
|
149
|
+
end
|
150
|
+
}
|
151
|
+
}
|
152
|
+
false
|
153
|
+
end
|
154
|
+
end
|
155
|
+
# Check if any attribute keys matching
|
156
|
+
class ContentHasKey
|
157
|
+
# Name in the configuration file
|
158
|
+
FILTER_NAME = 'content_has_key'
|
159
|
+
# Available mod modes
|
160
|
+
MODES = [:response_mod]
|
161
|
+
# Constructor
|
162
|
+
# +mode+:: resp or req mod
|
163
|
+
# +parameters+:: All parameters given in the configuration file
|
164
|
+
def initialize(_, parameters)
|
165
|
+
@params = parameters
|
166
|
+
end
|
167
|
+
# Execute plugin
|
168
|
+
# +data+:: ICAP data
|
169
|
+
def plugin(data)
|
170
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
171
|
+
doc.traverse do |node|
|
172
|
+
node.keys.each do |key|
|
173
|
+
@params.each do |attribute_key|
|
174
|
+
if key == attribute_key
|
175
|
+
data[:http_response_body] = '<html><body>Blocked attribute key</body></html>'
|
176
|
+
return true
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
180
|
+
end
|
181
|
+
false
|
182
|
+
end
|
183
|
+
end
|
184
|
+
# Check if the content of the tags includes any bad words
|
185
|
+
class ContentIncludes
|
186
|
+
# Name in the configuration file
|
187
|
+
FILTER_NAME = 'content_includes'
|
188
|
+
# Available mod modes
|
189
|
+
MODES = [:responese_mod]
|
190
|
+
# Constructor
|
191
|
+
# +mode+:: resp or req mod
|
192
|
+
# +parameters+:: All parameters given in the configuration file
|
193
|
+
def initialize(_, parameters)
|
194
|
+
@params = parameters
|
195
|
+
end
|
196
|
+
# Execute plugin
|
197
|
+
# +data+:: ICAP data
|
198
|
+
def plugin(data)
|
199
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
200
|
+
doc.traverse do |node|
|
201
|
+
@params.each { |phrase|
|
202
|
+
if node.text and node.text.include? phrase
|
203
|
+
data[:http_response_body] = '<html><body>Blocked content</body></html>'
|
204
|
+
return true
|
205
|
+
end
|
206
|
+
}
|
207
|
+
end
|
208
|
+
false
|
209
|
+
end
|
210
|
+
end
|
211
|
+
end
|
212
|
+
# Plugins for modify
|
213
|
+
module Plugins
|
214
|
+
# Send URLs to Virustotal and log into file
|
215
|
+
class VirusTotalURL
|
216
|
+
# Name in the configuration file
|
217
|
+
PLUGIN_NAME = 'send_url_to_virustotal'
|
218
|
+
# Available mod modes
|
219
|
+
MODES = [:request_mod]
|
220
|
+
|
221
|
+
# Constructor
|
222
|
+
# +mode+:: resp or req mod
|
223
|
+
# +parameters+:: All parameters given in the configuration file
|
224
|
+
def initialize(_, parameters)
|
225
|
+
@api_key = parameters[:api_key]
|
226
|
+
@log_file = File.expand_path(parameters[:log_file])
|
227
|
+
end
|
228
|
+
|
229
|
+
# Execute plugin
|
230
|
+
def plugin(data)
|
231
|
+
# Get address
|
232
|
+
uri = data[:http_request_header]['Host']
|
233
|
+
# Get result
|
234
|
+
ICAPrb::Filters.get_logger.debug("Sending uri #{uri.to_s} to Virustotal")
|
235
|
+
begin
|
236
|
+
results = Uirusu::VTUrl.query_report(@api_key, uri)
|
237
|
+
result = Uirusu::VTResult.new(uri, results)
|
238
|
+
# Write result
|
239
|
+
if result != nil
|
240
|
+
unless File.file?(@log_file)
|
241
|
+
FileUtils::touch @log_file
|
242
|
+
end
|
243
|
+
IO.write(@log_file, result.to_json.to_s.gsub('\n', '').gsub('\r', ''), mode: 'a')
|
244
|
+
end
|
245
|
+
rescue StandardError => e
|
246
|
+
ICAPrb::Filters.get_logger.warn('Virustotal error ' + e.message + e.backtrace.to_s)
|
247
|
+
end
|
248
|
+
false
|
249
|
+
end
|
250
|
+
end
|
251
|
+
# TODO at this time this should only work with request_mod
|
252
|
+
class ReplaceContent
|
253
|
+
# array of hashes in the form with the following fields
|
254
|
+
# match:: regular expression of url/
|
255
|
+
# content:: the content to use
|
256
|
+
# content_type:: 'set the content type'
|
257
|
+
attr_reader :replacement
|
258
|
+
# Constructor
|
259
|
+
# +mode+:: resp or req mod
|
260
|
+
# +parameters+:: All parameters given in the configuration file
|
261
|
+
def initialize(mode, parameters)
|
262
|
+
super
|
263
|
+
@replacement = []
|
264
|
+
end
|
265
|
+
|
266
|
+
# Name in the configuration file
|
267
|
+
PLUGIN_NAME = 'replace_content'
|
268
|
+
|
269
|
+
# Available mod modes
|
270
|
+
MODES = [:response_mod,:request_mod]
|
271
|
+
# Execute plugin
|
272
|
+
# +data+:: ICAP data
|
273
|
+
def plugin(data)
|
274
|
+
matching_replacements = @replacement.select {|replacement| data =~ replacement[:match] }
|
275
|
+
if matching_replacements.count > 0
|
276
|
+
matching_replacement = matching_replacements.first
|
277
|
+
data[:http_response_body] = ::ICAPrb::Server::ResponseBody.new(matching_replacement[:content])
|
278
|
+
if data[:http_response_header]
|
279
|
+
data[:http_response_header] =
|
280
|
+
::ICAPrb::Server::ResponseHeader.new(data[:http_response_header].http_version, 200)
|
281
|
+
elsif data[:http_request_header]
|
282
|
+
data[:http_response_header] =
|
283
|
+
::ICAPrb::Server::ResponseHeader.new(data[:http_request_header].http_version, 200)
|
284
|
+
else
|
285
|
+
data[:http_response_header] = ::ICAPrb::Server::ResponseHeader.new('1.1', 200)
|
286
|
+
end
|
287
|
+
|
288
|
+
data[:http_response_header]['Content-Type'] = matching_replacement[:content_type]
|
289
|
+
data[:http_response_header]['Content-Length'] = matching_replacement[:content].length
|
290
|
+
data.delete(:http_request_header) if data[:http_request_header]
|
291
|
+
return true
|
292
|
+
end
|
293
|
+
false
|
294
|
+
end
|
295
|
+
end
|
296
|
+
|
297
|
+
#This class is used to delete common implementations of facebook buttons
|
298
|
+
class RemoveFB
|
299
|
+
# +mode+:: resp or req mod
|
300
|
+
# +parameters+:: All parameters given in the configuration file
|
301
|
+
def initialize(_, _)
|
302
|
+
end
|
303
|
+
|
304
|
+
# class name
|
305
|
+
PLUGIN_NAME = 'remove_fb'
|
306
|
+
|
307
|
+
# Available mod modes
|
308
|
+
MODES = [:response_mod]
|
309
|
+
|
310
|
+
#execute the plug in
|
311
|
+
def plugin(data)
|
312
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
313
|
+
mod = false
|
314
|
+
|
315
|
+
# removes all iframes where the source matches
|
316
|
+
doc.search('iframe').each do |iframe|
|
317
|
+
if iframe['src'].match(/facebook\.com\/.*like\.php/i)
|
318
|
+
iframe.remove
|
319
|
+
mod = true
|
320
|
+
end
|
321
|
+
end
|
322
|
+
|
323
|
+
# removes all matching links
|
324
|
+
doc.search('a').each do |a|
|
325
|
+
if a['href'].to_s.match(/www\.facebook\.com\/sharer\.php|www\.facebook\.com\/sharer\/sharer\.php/)
|
326
|
+
a.remove
|
327
|
+
mod = true
|
328
|
+
end
|
329
|
+
end
|
330
|
+
|
331
|
+
# removes the facebook like-button element
|
332
|
+
doc.search('div.fb-like').each do |src|
|
333
|
+
src.remove
|
334
|
+
mod = true
|
335
|
+
end
|
336
|
+
|
337
|
+
# removes the facebook share-button element
|
338
|
+
doc.search('div.fb-share-button').each do |src|
|
339
|
+
src.remove
|
340
|
+
mod = true
|
341
|
+
end
|
342
|
+
|
343
|
+
# removes the facebook comments element
|
344
|
+
doc.search('div.fb-comments').each do |src|
|
345
|
+
src.remove
|
346
|
+
mod = true
|
347
|
+
end
|
348
|
+
|
349
|
+
# removes all scripts where the content matches
|
350
|
+
doc.css('script').each do |script|
|
351
|
+
if script.content.match(/connect\.facebook\.net\//i)
|
352
|
+
script.remove
|
353
|
+
mod = true
|
354
|
+
end
|
355
|
+
end
|
356
|
+
|
357
|
+
|
358
|
+
data[:http_response_body] = doc.to_s if mod
|
359
|
+
mod
|
360
|
+
end
|
361
|
+
end
|
362
|
+
|
363
|
+
#This class is used to delete common implementations of the Twitter-Share-Button
|
364
|
+
class RemoveRetweetButton
|
365
|
+
# +mode+:: resp or req mod
|
366
|
+
# +parameters+:: All parameters given in the configuration file
|
367
|
+
def initialize(_, _)
|
368
|
+
end
|
369
|
+
|
370
|
+
#class name
|
371
|
+
PLUGIN_NAME = 'remove_retweet_button'
|
372
|
+
|
373
|
+
# Available mod modes
|
374
|
+
MODES = [:response_mod]
|
375
|
+
# Execute plugin
|
376
|
+
# +data+:: ICAP data
|
377
|
+
def plugin(data)
|
378
|
+
mod = false
|
379
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
380
|
+
|
381
|
+
# removes twitter share button elements
|
382
|
+
doc.search('a.twitter-share-button').each do |src|
|
383
|
+
src.remove
|
384
|
+
mod = true
|
385
|
+
end
|
386
|
+
|
387
|
+
# removes twitter timeline elements
|
388
|
+
doc.search('a.twitter-timeline').each do |src|
|
389
|
+
src.remove
|
390
|
+
mod = true
|
391
|
+
end
|
392
|
+
|
393
|
+
# removes all matching links
|
394
|
+
doc.search('a').each do |a|
|
395
|
+
if a['href'].to_s.match(/twitter\.com\/share\?/i)
|
396
|
+
a.remove
|
397
|
+
mod = true
|
398
|
+
end
|
399
|
+
if a['href'].to_s.match(/twitter\.com\/intent\//i)
|
400
|
+
a.remove
|
401
|
+
mod = true
|
402
|
+
end
|
403
|
+
end
|
404
|
+
|
405
|
+
# removes all scripts where the content matches
|
406
|
+
doc.css('script').each do |script|
|
407
|
+
if script.content.match(/\/\/platform\.twitter\.com\/widgets\.js/i)
|
408
|
+
script.remove
|
409
|
+
mod = true
|
410
|
+
end
|
411
|
+
end
|
412
|
+
|
413
|
+
# removes all iframes where the source matches
|
414
|
+
doc.search('iframe').each do |iframe|
|
415
|
+
if iframe['src'].match(/platform\.twitter\.com\/widgets/i)
|
416
|
+
iframe.remove
|
417
|
+
mod = true
|
418
|
+
end
|
419
|
+
end
|
420
|
+
|
421
|
+
data[:http_response_body] = doc.to_s if mod
|
422
|
+
mod
|
423
|
+
end
|
424
|
+
end
|
425
|
+
|
426
|
+
#Plug in to remove the Google PlusOne Button
|
427
|
+
class RemovePlusOneButton
|
428
|
+
# +mode+:: resp or req mod
|
429
|
+
# +parameters+:: All parameters given in the configuration file
|
430
|
+
def initialize(_, _)
|
431
|
+
end
|
432
|
+
|
433
|
+
# class name
|
434
|
+
PLUGIN_NAME = 'remove_plusone_button'
|
435
|
+
|
436
|
+
# Available mod modes
|
437
|
+
MODES = [:response_mod]
|
438
|
+
|
439
|
+
# execute the plug in
|
440
|
+
def plugin(data)
|
441
|
+
mod = false
|
442
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
443
|
+
|
444
|
+
# removes all g-plusone elements
|
445
|
+
doc.search('div.g-plusone').each do |src|
|
446
|
+
src.remove
|
447
|
+
mod = true
|
448
|
+
end
|
449
|
+
|
450
|
+
# removes all matching links
|
451
|
+
doc.search('a').each do |a|
|
452
|
+
if a['href'].to_s.match(/plus\.google\.com\/share\?/i)
|
453
|
+
end
|
454
|
+
end
|
455
|
+
|
456
|
+
# removes all scripts where the source or the content matches
|
457
|
+
doc.css('script').each do |script|
|
458
|
+
if script['src'] == 'https://apis.google.com/js/platform.js' || script.content.match(/apis\.google\.com/i)
|
459
|
+
script.remove
|
460
|
+
mod = true
|
461
|
+
end
|
462
|
+
end
|
463
|
+
|
464
|
+
# removes all iframes where the source matches
|
465
|
+
doc.search('iframe').each do |iframe|
|
466
|
+
if iframe['src'].match(/apis\.google\.com/i)
|
467
|
+
iframe.remove
|
468
|
+
mod = true
|
469
|
+
end
|
470
|
+
end
|
471
|
+
|
472
|
+
data[:http_response_body] = doc.to_s if mod
|
473
|
+
mod
|
474
|
+
end
|
475
|
+
end
|
476
|
+
|
477
|
+
# remove the two click solution provided by heise.de
|
478
|
+
class RemoveSocialSharePrivacy
|
479
|
+
# +mode+:: resp or req mod
|
480
|
+
# +parameters+:: All parameters given in the configuration file
|
481
|
+
def initialize(_, _)
|
482
|
+
end
|
483
|
+
|
484
|
+
# class name
|
485
|
+
PLUGIN_NAME = 'remove_social_share_privacy'
|
486
|
+
|
487
|
+
# Available mod modes
|
488
|
+
MODES = [:response_mod]
|
489
|
+
|
490
|
+
# execute the plug in
|
491
|
+
def plugin(data)
|
492
|
+
mod = false
|
493
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
494
|
+
|
495
|
+
# removes all socialshareprivacy elements
|
496
|
+
doc.css('#socialshareprivacy').each do |src|
|
497
|
+
src.remove
|
498
|
+
mod = true
|
499
|
+
end
|
500
|
+
|
501
|
+
# removes all scripts where the source or the content matches
|
502
|
+
doc.css('script').each do |script|
|
503
|
+
if script.content.match(/socialshareprivacy\.css/i) || (script['src'] == 'jquery.socialshareprivacy.js')
|
504
|
+
script.remove
|
505
|
+
mod = true
|
506
|
+
end
|
507
|
+
end
|
508
|
+
data[:http_response_body] = doc.to_s if mod
|
509
|
+
mod
|
510
|
+
end
|
511
|
+
end
|
512
|
+
|
513
|
+
#Plug in to remove all Google AdWords
|
514
|
+
class RemoveGoogleAdWords
|
515
|
+
# +mode+:: resp or req mod
|
516
|
+
# +parameters+:: All parameters given in the configuration file
|
517
|
+
def initialize(_, _)
|
518
|
+
end
|
519
|
+
|
520
|
+
# class name
|
521
|
+
PLUGIN_NAME = 'remove_google_ad_words'
|
522
|
+
|
523
|
+
# Available mod modes
|
524
|
+
MODES = [:response_mod]
|
525
|
+
|
526
|
+
# execute the plug in
|
527
|
+
# google AdWords are loaded with JSON files
|
528
|
+
# the plugin looks for the source record of google Adwords in those
|
529
|
+
# files and deletes them
|
530
|
+
def plugin(data)
|
531
|
+
mod = false
|
532
|
+
joined = ''
|
533
|
+
|
534
|
+
if data[:icap_data][:request_line][:uri].to_s.match(/www\.google\..*/i)
|
535
|
+
if data[:http_response_header]['Content-Type']=='application/json'
|
536
|
+
splitted = data[:http_response_body].split('/*""*/')
|
537
|
+
parsed = splitted.map {|x| JSON.parse x if x.length > 2}
|
538
|
+
parsed.each do |x|
|
539
|
+
if x && ( x['d'].include? 'www.googleadservices.com')
|
540
|
+
x['d'] = ''
|
541
|
+
mod = true
|
542
|
+
end
|
543
|
+
end
|
544
|
+
if mod
|
545
|
+
parsed.each do |x|
|
546
|
+
joined = joined + x.to_json + '/*""*/'
|
547
|
+
end
|
548
|
+
data[:http_response_body] = joined
|
549
|
+
end
|
550
|
+
end
|
551
|
+
end
|
552
|
+
mod
|
553
|
+
end
|
554
|
+
end
|
555
|
+
|
556
|
+
# Plug in to remove the Google Webads
|
557
|
+
class RemoveGoogleWebAds
|
558
|
+
# +mode+:: resp or req mod
|
559
|
+
# +parameters+:: All parameters given in the configuration file
|
560
|
+
def initialize(_, _)
|
561
|
+
end
|
562
|
+
|
563
|
+
# class name
|
564
|
+
PLUGIN_NAME = 'remove_google_web_ads'
|
565
|
+
|
566
|
+
# Available mod modes
|
567
|
+
MODES = [:response_mod]
|
568
|
+
|
569
|
+
# execute the plug in
|
570
|
+
def plugin(data)
|
571
|
+
mod = false
|
572
|
+
|
573
|
+
#stops loading from specific hosts
|
574
|
+
if data[:http_request_header]['Host'].to_s.match(/googleads|googlesyndicat|googleadservice|\.doubleclick\./i)
|
575
|
+
data[:http_response_body] = ''
|
576
|
+
mod = true
|
577
|
+
end
|
578
|
+
|
579
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
580
|
+
|
581
|
+
#ICAPrb::Filters.get_logger.debug('content: ' + content = data[:http_response_body].to_s)
|
582
|
+
|
583
|
+
#delete divs where div.class contains keyword
|
584
|
+
doc.search('div').each do |src|
|
585
|
+
if src['class'].to_s.match(/ads|google_ads|google_companion|google-companion|googlesyndicat/i) || src['id'].to_s.match(/google_ads/i)
|
586
|
+
src.remove
|
587
|
+
mod = true
|
588
|
+
end
|
589
|
+
end
|
590
|
+
|
591
|
+
#delete hyperlinks where href contains keyword
|
592
|
+
doc.search('a').each do |a|
|
593
|
+
if a['href'].to_s.match(/googleads|googleadservice|googlesyndicat|\.doubleclick\./i)
|
594
|
+
a.remove
|
595
|
+
mod = true
|
596
|
+
end
|
597
|
+
end
|
598
|
+
|
599
|
+
#delete scripts where src or content matches keyword
|
600
|
+
doc.search('script').each do |script|
|
601
|
+
if script.content.match(/googleads|.*googlesyndicat.*|googleadservice|\.doubleclick\./i) || script['src'].to_s.match(/googleads|googlesyndicat|googleadservice|\.doubleclick\./i)
|
602
|
+
script.remove
|
603
|
+
mod = true
|
604
|
+
end
|
605
|
+
end
|
606
|
+
|
607
|
+
#delete images where the source matches
|
608
|
+
doc.search('img').each do |img|
|
609
|
+
if img['src'].to_s.match(/googleads\.|\.doubleclick\./i)
|
610
|
+
img.remove
|
611
|
+
mod = true
|
612
|
+
end
|
613
|
+
end
|
614
|
+
|
615
|
+
# removes google Ad Elements
|
616
|
+
doc.search('gwd-doubleclick').each do |gwd|
|
617
|
+
gwd.remove
|
618
|
+
mod = true
|
619
|
+
end
|
620
|
+
|
621
|
+
# removes google Ad Elements
|
622
|
+
doc.search('gwd-genericad').each do |gwd|
|
623
|
+
gwd.remove
|
624
|
+
mod = true
|
625
|
+
end
|
626
|
+
|
627
|
+
data[:http_response_body] = doc.to_s if mod
|
628
|
+
|
629
|
+
mod
|
630
|
+
end
|
631
|
+
end
|
632
|
+
|
633
|
+
# Plug in to remove the Adition Webads
|
634
|
+
class RemoveAditionWebAds
|
635
|
+
# +mode+:: resp or req mod
|
636
|
+
# +parameters+:: All parameters given in the configuration file
|
637
|
+
def initialize(_, _)
|
638
|
+
end
|
639
|
+
|
640
|
+
# class name
|
641
|
+
PLUGIN_NAME = 'remove_adition_web_ads'
|
642
|
+
|
643
|
+
# Available mod modes
|
644
|
+
MODES = [:response_mod]
|
645
|
+
|
646
|
+
# execute the plug in
|
647
|
+
# removes all links and scripts where the source or the content matches
|
648
|
+
def plugin(data)
|
649
|
+
mod = false
|
650
|
+
|
651
|
+
#stops loading from specific hosts
|
652
|
+
if data[:http_request_header]['Host'].to_s.match(/\.adfarm[0-9]+\.adition/i)
|
653
|
+
data[:http_response_body] = ''
|
654
|
+
mod = true
|
655
|
+
end
|
656
|
+
|
657
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
658
|
+
|
659
|
+
doc.search('a').each do |a|
|
660
|
+
if a['href'].to_s.match(/\.adfarm[0-9]+\.adition\.com/i)
|
661
|
+
a.remove
|
662
|
+
mod = true
|
663
|
+
end
|
664
|
+
end
|
665
|
+
|
666
|
+
doc.search('script').each do |script|
|
667
|
+
if script['src'].to_s.match(/\.adfarm[0-9]+\.adition/i) || script.content.to_s.match(/adition|imagesrv\.adition\.com\/js\/srp\.js/i)
|
668
|
+
script.remove
|
669
|
+
mod = true
|
670
|
+
end
|
671
|
+
end
|
672
|
+
data[:http_response_body] = doc.to_s if mod
|
673
|
+
mod
|
674
|
+
end
|
675
|
+
end
|
676
|
+
|
677
|
+
# Plug in to remove the Nuggad Webads
|
678
|
+
class RemoveNuggadWebAds
|
679
|
+
# +mode+:: resp or req mod
|
680
|
+
# +parameters+:: All parameters given in the configuration file
|
681
|
+
def initialize(_, _)
|
682
|
+
end
|
683
|
+
|
684
|
+
# class name
|
685
|
+
PLUGIN_NAME = 'remove_nuggad_web_ads'
|
686
|
+
|
687
|
+
# Available mod modes
|
688
|
+
MODES = [:response_mod]
|
689
|
+
|
690
|
+
# execute the plug in
|
691
|
+
# removes all scripts where the source or the content matches
|
692
|
+
def plugin(data)
|
693
|
+
mod = false
|
694
|
+
|
695
|
+
#stops loading from specific host
|
696
|
+
if data[:http_request_header]['Host'].to_s.match(/\.nuggad\.net/i)
|
697
|
+
data[:http_response_body] = ''
|
698
|
+
mod = true
|
699
|
+
end
|
700
|
+
|
701
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
702
|
+
|
703
|
+
doc.search('script').each do |script|
|
704
|
+
if script['src'].to_s.match(/\.nuggad\.net/i) || script.content.match(/\.nuggad\.net/i)
|
705
|
+
script.remove
|
706
|
+
mod = true
|
707
|
+
end
|
708
|
+
end
|
709
|
+
data[:http_response_body] = doc.to_s if mod
|
710
|
+
mod
|
711
|
+
end
|
712
|
+
end
|
713
|
+
|
714
|
+
# Plug in to remove the Google Tag Manager
|
715
|
+
class RemoveGoogleTagManager
|
716
|
+
# +mode+:: resp or req mod
|
717
|
+
# +parameters+:: All parameters given in the configuration file
|
718
|
+
def initialize(_, _)
|
719
|
+
end
|
720
|
+
|
721
|
+
# class name
|
722
|
+
PLUGIN_NAME = 'remove_google_tag_manager'
|
723
|
+
|
724
|
+
# Available mod modes
|
725
|
+
MODES = [:response_mod]
|
726
|
+
|
727
|
+
# execute the plug in
|
728
|
+
# removes all scripts where the source or the content matches
|
729
|
+
def plugin(data)
|
730
|
+
mod = false
|
731
|
+
|
732
|
+
#stops loading from specific hosts
|
733
|
+
if data[:http_request_header]['Host'].to_s.match(/googletagservices\.com/i)
|
734
|
+
data[:http_response_body] = ''
|
735
|
+
mod = true
|
736
|
+
end
|
737
|
+
|
738
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
739
|
+
|
740
|
+
doc.search('script').each do |script|
|
741
|
+
if script['src'].to_s.match(/www\.googletagservices\.com\/tag\/js\/gpt\.js/i) || script.content.match(/googletagservices|googletag/i)
|
742
|
+
script.remove
|
743
|
+
mod = true
|
744
|
+
end
|
745
|
+
end
|
746
|
+
data[:http_response_body] = doc.to_s if mod
|
747
|
+
mod
|
748
|
+
end
|
749
|
+
end
|
750
|
+
|
751
|
+
# Plug in to remove OEWA tracker.
|
752
|
+
class RemoveOewaTracker
|
753
|
+
# +mode+:: resp or req mod
|
754
|
+
# +parameters+:: All parameters given in the configuration file
|
755
|
+
def initialize(_, _)
|
756
|
+
end
|
757
|
+
|
758
|
+
# class name
|
759
|
+
PLUGIN_NAME = 'remove_oewa_tracker'
|
760
|
+
|
761
|
+
# Available mod modes
|
762
|
+
MODES = [:response_mod]
|
763
|
+
|
764
|
+
# execute the plug in
|
765
|
+
# removes all scripts where the source or the content matches
|
766
|
+
def plugin(data)
|
767
|
+
mod = false
|
768
|
+
|
769
|
+
#stops loading from specific host
|
770
|
+
if data[:http_request_header]['Host'].to_s.match(/oewabox\.at/i)
|
771
|
+
data[:http_response_body] = ''
|
772
|
+
mod = true
|
773
|
+
end
|
774
|
+
|
775
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
776
|
+
|
777
|
+
doc.search('script').each do |script|
|
778
|
+
if script['src'].to_s.match(/oewabox\.at\/oewa\.js/i) || script.content.match(/oewabox\.at\/oewa\.js/i)
|
779
|
+
script.remove
|
780
|
+
mod = true
|
781
|
+
end
|
782
|
+
end
|
783
|
+
data[:http_response_body] = doc.to_s if mod
|
784
|
+
mod
|
785
|
+
end
|
786
|
+
end
|
787
|
+
|
788
|
+
# Plug in to remove the Piwik Tracker
|
789
|
+
class RemovePiwikTracker
|
790
|
+
# +mode+:: resp or req mod
|
791
|
+
# +parameters+:: All parameters given in the configuration file
|
792
|
+
def initialize(_, _)
|
793
|
+
end
|
794
|
+
|
795
|
+
# class name
|
796
|
+
PLUGIN_NAME = 'remove_piwik_tracker'
|
797
|
+
|
798
|
+
# Available mod modes
|
799
|
+
MODES = [:response_mod]
|
800
|
+
|
801
|
+
# execute the plug in
|
802
|
+
# removes all scripts where the source or the content matches
|
803
|
+
def plugin(data)
|
804
|
+
mod = false
|
805
|
+
|
806
|
+
#stops loading from specific host
|
807
|
+
if data[:http_request_header]['Host'].to_s.match(/demo[0-9]+\.piwik\.org/i)
|
808
|
+
data[:http_response_body] = ''
|
809
|
+
mod = true
|
810
|
+
end
|
811
|
+
|
812
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
813
|
+
|
814
|
+
doc.search('script').each do |script|
|
815
|
+
if script['src'].to_s.match(/piwik\.js/i) || script.content.match(/piwik\.js/i)
|
816
|
+
script.remove
|
817
|
+
mod = true
|
818
|
+
end
|
819
|
+
end
|
820
|
+
data[:http_response_body] = doc.to_s if mod
|
821
|
+
mod
|
822
|
+
end
|
823
|
+
end
|
824
|
+
|
825
|
+
# Plug in to remove the Google Analytics tracker
|
826
|
+
class RemoveGoogleAnaytics
|
827
|
+
# +mode+:: resp or req mod
|
828
|
+
# +parameters+:: All parameters given in the configuration file
|
829
|
+
def initialize(_, _)
|
830
|
+
end
|
831
|
+
|
832
|
+
# class name
|
833
|
+
PLUGIN_NAME = 'remove_google_analytics'
|
834
|
+
|
835
|
+
# Available mod modes
|
836
|
+
MODES = [:response_mod]
|
837
|
+
|
838
|
+
# execute the plug in
|
839
|
+
# removes all scripts where the source or the content matches
|
840
|
+
def plugin(data)
|
841
|
+
mod = false
|
842
|
+
|
843
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
844
|
+
|
845
|
+
doc.search('script').each do |script|
|
846
|
+
if script['src'].to_s.match(/\/analytics\.js/i) || script.content.match(/\/analytics\.js/i)
|
847
|
+
script.remove
|
848
|
+
mod = true
|
849
|
+
end
|
850
|
+
end
|
851
|
+
data[:http_response_body] = doc.to_s if mod
|
852
|
+
mod
|
853
|
+
end
|
854
|
+
end
|
855
|
+
|
856
|
+
# Plug in to remove the ioam.de tracker.
|
857
|
+
class RemoveIoam
|
858
|
+
# +mode+:: resp or req mod
|
859
|
+
# +parameters+:: All parameters given in the configuration file
|
860
|
+
def initialize(_, _)
|
861
|
+
end
|
862
|
+
|
863
|
+
# class name
|
864
|
+
PLUGIN_NAME = 'remove_ioam'
|
865
|
+
|
866
|
+
# Available mod modes
|
867
|
+
MODES = [:response_mod]
|
868
|
+
|
869
|
+
# execute the plug in
|
870
|
+
# removes all scripts where the source or the content matches
|
871
|
+
def plugin(data)
|
872
|
+
mod = false
|
873
|
+
|
874
|
+
#stops loading from specific host
|
875
|
+
if data[:http_request_header]['Host'].to_s.match(/script\.ioam.de/i)
|
876
|
+
data[:http_response_body] = ''
|
877
|
+
mod = true
|
878
|
+
end
|
879
|
+
|
880
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
881
|
+
|
882
|
+
doc.search('script').each do |script|
|
883
|
+
if script['src'].to_s.match(/script\.ioam.de\/iam\.js/i) || script.content.match(/ioam.de\/iam\.js/i)
|
884
|
+
script.remove
|
885
|
+
mod = true
|
886
|
+
end
|
887
|
+
end
|
888
|
+
data[:http_response_body] = doc.to_s if mod
|
889
|
+
mod
|
890
|
+
end
|
891
|
+
end
|
892
|
+
|
893
|
+
# Plug in to remove the Webtrekk Tracker
|
894
|
+
class RemoveWebtrekk# Constructor
|
895
|
+
# +mode+:: resp or req mod
|
896
|
+
# +parameters+:: All parameters given in the configuration file
|
897
|
+
def initialize(_, _)
|
898
|
+
end
|
899
|
+
|
900
|
+
# class name
|
901
|
+
PLUGIN_NAME = 'remove_webtrekk'
|
902
|
+
|
903
|
+
# Available mod modes
|
904
|
+
MODES = [:response_mod]
|
905
|
+
|
906
|
+
# execute the plug in
|
907
|
+
# removes all scripts where the source or the content matches
|
908
|
+
def plugin(data)
|
909
|
+
mod = false
|
910
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
911
|
+
|
912
|
+
doc.search('script').each do |script|
|
913
|
+
if script['src'].to_s.match(/webtrekk[\S]*\.js/i) || script.content.match(/webtrekk[\S]*\.js/i)
|
914
|
+
script.remove
|
915
|
+
mod = true
|
916
|
+
end
|
917
|
+
end
|
918
|
+
data[:http_response_body] = doc.to_s if mod
|
919
|
+
mod
|
920
|
+
end
|
921
|
+
end
|
922
|
+
|
923
|
+
# Plug in to remove the Visual Revenue Tracker
|
924
|
+
class RemoveVisualRevenue
|
925
|
+
# Constructor
|
926
|
+
# +mode+:: resp or req mod
|
927
|
+
# +parameters+:: All parameters given in the configuration file
|
928
|
+
def initialize(_, _)
|
929
|
+
end
|
930
|
+
|
931
|
+
# class name
|
932
|
+
PLUGIN_NAME = 'remove_visual_revenue'
|
933
|
+
|
934
|
+
# Available mod modes
|
935
|
+
MODES = [:response_mod]
|
936
|
+
|
937
|
+
# execute the plug in
|
938
|
+
# removes all scripts where the source or the content matches
|
939
|
+
def plugin(data)
|
940
|
+
mod = false
|
941
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
942
|
+
|
943
|
+
doc.search('script').each do |script|
|
944
|
+
if script['src'].to_s.match(/visualrevenue\.com\/vrs\.js/i) || script.content.match(/visualrevenue\.com\/vrs\.js/i)
|
945
|
+
script.remove
|
946
|
+
mod = true
|
947
|
+
end
|
948
|
+
end
|
949
|
+
data[:http_response_body] = doc.to_s if mod
|
950
|
+
mod
|
951
|
+
end
|
952
|
+
end
|
953
|
+
end
|
954
|
+
end
|