icaprb-filter 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.rspec +2 -0
- data/.travis.yml +4 -0
- data/Gemfile +4 -0
- data/LICENSE +25 -0
- data/README.md +68 -0
- data/README.rdoc +22 -0
- data/Rakefile +12 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/bin/start_server.rb +40 -0
- data/icaprb-filter.gemspec +33 -0
- data/lib/icaprb/filter.rb +189 -0
- data/lib/icaprb/filter/service.rb +107 -0
- data/lib/icaprb/filter/solution.rb +954 -0
- data/lib/icaprb/filter/steganography.rb +91 -0
- data/lib/icaprb/filter/version.rb +8 -0
- data/samples/block_samples/check_hashes_test.rb +7 -0
- data/samples/block_samples/check_hashes_test.txt +2 -0
- data/samples/block_samples/content_has_key_test.html +10 -0
- data/samples/block_samples/content_has_key_test.rb +6 -0
- data/samples/block_samples/content_includes_test.html +10 -0
- data/samples/block_samples/content_includes_test.rb +6 -0
- data/samples/block_samples/nothing_to_filter_test.html +10 -0
- data/samples/block_samples/url_contains_test.rb +6 -0
- data/samples/custom_samples/virustotal_config.rb +6 -0
- metadata +205 -0
@@ -0,0 +1,107 @@
|
|
1
|
+
require 'icaprb/server'
|
2
|
+
require 'icaprb/filter'
|
3
|
+
|
4
|
+
# nodoc
|
5
|
+
module ICAPrb
|
6
|
+
# Server
|
7
|
+
module Server
|
8
|
+
# The filter service, reference in your startup script
|
9
|
+
class FilterService < ICAPrb::Server::Services::ServiceBase
|
10
|
+
# Load configuration for filter service
|
11
|
+
# +path+:: Path to the configuration file
|
12
|
+
def initialize(path, timeout=nil)
|
13
|
+
super('filter',[:request_mod, :response_mod],nil,60,nil,nil,nil,1000)
|
14
|
+
@timeout = timeout
|
15
|
+
@request_list, @response_list = ::ICAPrb::Filter.load_filters(path)
|
16
|
+
end
|
17
|
+
# Process a new filter request
|
18
|
+
# +icap_server+: ICAP server data
|
19
|
+
# +ip+:: IP
|
20
|
+
# +socket+:: Socket
|
21
|
+
# +data+:: Data for the current request
|
22
|
+
def process_request(icap_server,ip,socket,data)
|
23
|
+
logger = icap_server.logger
|
24
|
+
ICAPrb::Filter.set_logger(logger)
|
25
|
+
ICAPrb::Filters.set_logger(logger)
|
26
|
+
logger.debug '==========================================='
|
27
|
+
logger.debug 'Start processing data via filter service...'
|
28
|
+
logger.debug 'Request url: ' + data[:http_request_header]["Host"].to_s
|
29
|
+
|
30
|
+
#logger.debug data
|
31
|
+
# Go through all filters for resp or req
|
32
|
+
begin
|
33
|
+
response = ::ICAPrb::Server::Response.new
|
34
|
+
is_modified = false
|
35
|
+
if data[:icap_data][:request_line][:icap_method] == :request_mod
|
36
|
+
logger.debug('Event is REQUEST')
|
37
|
+
@request_list.each do |entry|
|
38
|
+
logger.debug('Executing filter: ' + entry[:name].to_s)
|
39
|
+
filter_return_value = entry[:object].plugin(data)
|
40
|
+
if [true, false].include? filter_return_value
|
41
|
+
is_modified = (filter_return_value || is_modified)
|
42
|
+
logger.debug("Filter return value: " + filter_return_value.to_s)
|
43
|
+
else
|
44
|
+
logger.error("Filter " + entry[:name] + ' has illegal return value')
|
45
|
+
end
|
46
|
+
end
|
47
|
+
elsif data[:icap_data][:request_line][:icap_method] == :response_mod
|
48
|
+
logger.debug('Event is RESPONSE')
|
49
|
+
@response_list.each do |entry|
|
50
|
+
logger.debug('Executing plugin: ' + entry[:name].to_s)
|
51
|
+
plugin_return_value = entry[:object].plugin(data)
|
52
|
+
if [true, false].include? plugin_return_value
|
53
|
+
is_modified = (plugin_return_value || is_modified)
|
54
|
+
logger.debug("Plugin return value: " + plugin_return_value.to_s)
|
55
|
+
else
|
56
|
+
logger.error("Plugin " + entry[:name] + ' has illegal return value')
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
if is_modified
|
61
|
+
logger.debug('Data was modified, sending everything')
|
62
|
+
else
|
63
|
+
logger.debug('Data is unmodified, sending status code 204')
|
64
|
+
end
|
65
|
+
# Nothing changed
|
66
|
+
unless is_modified
|
67
|
+
# Send 204 (unmodified) flag, no content required
|
68
|
+
response.icap_status_code = 204
|
69
|
+
response.write_headers_to_socket socket
|
70
|
+
return
|
71
|
+
end
|
72
|
+
# Something changed
|
73
|
+
if data[:icap_data][:request_line][:icap_method] == :response_mod
|
74
|
+
http_header = data[:http_response_header]
|
75
|
+
if data[:http_response_body]
|
76
|
+
http_body = ICAPrb::Server::ResponseBody.new(data[:http_response_body])
|
77
|
+
http_header['Content-Length'] = http_body.length
|
78
|
+
end
|
79
|
+
else
|
80
|
+
http_header = data[:http_request_header]
|
81
|
+
if data[:http_request_body]
|
82
|
+
http_body = ICAPrb::Server::RequestBody.new(data[:http_request_body])
|
83
|
+
http_header['Content-Length'] = http_body.length
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
response.components << http_header
|
88
|
+
response.components << http_body if http_body
|
89
|
+
response.icap_status_code = 200
|
90
|
+
response.write_headers_to_socket socket
|
91
|
+
socket.write(http_body.to_chunk)
|
92
|
+
::ICAPrb::Server::Response.send_last_chunk(socket,false)
|
93
|
+
rescue StandardError => error
|
94
|
+
logger.warn 'Error in filter framework'
|
95
|
+
logger.warn error.message
|
96
|
+
logger.warn error.backtrace
|
97
|
+
logger.warn error.backtrace_locations
|
98
|
+
# Don't anger the consumer
|
99
|
+
response = ::ICAPrb::Server::Response.new
|
100
|
+
response.icap_status_code = 204
|
101
|
+
response.write_headers_to_socket socket
|
102
|
+
end
|
103
|
+
logger.debug '==========================================='
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
@@ -0,0 +1,954 @@
|
|
1
|
+
require 'pdf/reader'
|
2
|
+
require 'nokogiri'
|
3
|
+
require 'icaprb/server'
|
4
|
+
require 'rest-client'
|
5
|
+
require 'json'
|
6
|
+
require 'digest'
|
7
|
+
require 'uirusu'
|
8
|
+
|
9
|
+
# ICAPrb
|
10
|
+
module ICAPrb
|
11
|
+
# Methods for filters
|
12
|
+
class Filters
|
13
|
+
# Set logger for filter
|
14
|
+
# +logger+:: Logger object
|
15
|
+
def self.set_logger(logger)
|
16
|
+
@logger = logger
|
17
|
+
end
|
18
|
+
|
19
|
+
# Get logger
|
20
|
+
def self.get_logger
|
21
|
+
@logger
|
22
|
+
end
|
23
|
+
|
24
|
+
# Generate class reference from string
|
25
|
+
# +str+:: String to get the class from
|
26
|
+
def self.class_from_string(str)
|
27
|
+
str.split('::').inject(Object) do |mod, class_name|
|
28
|
+
mod.const_get(class_name)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
# Get all available filters in the namespace ICAPrb::FilterSolutions::
|
32
|
+
def self.get_filters
|
33
|
+
filter_symbols = FilterSolutions.constants.select{ |c| FilterSolutions.const_get(c).is_a? Class }
|
34
|
+
filters = []
|
35
|
+
filter_symbols.each { |filter_symbol|
|
36
|
+
symbol_path = 'ICAPrb::FilterSolutions::' + filter_symbol.to_s
|
37
|
+
current_class = self.class_from_string(symbol_path)
|
38
|
+
filters << {:class => current_class, :name => current_class::FILTER_NAME, :modes => current_class::MODES}
|
39
|
+
}
|
40
|
+
filters
|
41
|
+
end
|
42
|
+
|
43
|
+
# Get all available plugins in the namespace ICAPrb::Plugins::
|
44
|
+
def self.get_plugins
|
45
|
+
plugin_symbols = Plugins.constants.select{ |c| Plugins.const_get(c).is_a? Class }
|
46
|
+
plugins = []
|
47
|
+
plugin_symbols.each { |plugin_symbol|
|
48
|
+
symbol_path = 'ICAPrb::Plugins::' + plugin_symbol.to_s
|
49
|
+
current_class = self.class_from_string(symbol_path)
|
50
|
+
plugins << {:class => current_class, :name => current_class::PLUGIN_NAME, :modes =>current_class::MODES}
|
51
|
+
}
|
52
|
+
plugins
|
53
|
+
end
|
54
|
+
end
|
55
|
+
# Contains block rule definitions
|
56
|
+
module FilterSolutions
|
57
|
+
# Check for hashes and block if anything matches
|
58
|
+
class HashedContent
|
59
|
+
# Name in the configuration file
|
60
|
+
FILTER_NAME = 'check_hashes'
|
61
|
+
# Available mod modes
|
62
|
+
MODES = [:response_mod]
|
63
|
+
# Constructor
|
64
|
+
# +mode+:: resp or req mod
|
65
|
+
# +parameters+:: All parameters given in the configuration file
|
66
|
+
def initialize(_, parameters)
|
67
|
+
@params = []
|
68
|
+
file_path = parameters[0]
|
69
|
+
File.foreach(File.expand_path(file_path)) do |line|
|
70
|
+
line = line.chomp.gsub('\n', '').gsub('\r', '')
|
71
|
+
@params << line
|
72
|
+
end
|
73
|
+
end
|
74
|
+
# Execute plugin
|
75
|
+
# +data+:: ICAP data
|
76
|
+
def plugin(data)
|
77
|
+
if data[:http_response_body] == nil
|
78
|
+
ICAPrb::Filters.get_logger.debug('Nothing to hash')
|
79
|
+
return false
|
80
|
+
end
|
81
|
+
hash = Digest::SHA256.hexdigest(data[:http_response_body]).to_s
|
82
|
+
ICAPrb::Filters.get_logger.debug('Calculated hash: ' + hash.to_s)
|
83
|
+
@params.each do |other_hash|
|
84
|
+
if hash.eql? other_hash
|
85
|
+
# Block by telling the user that this is blocked
|
86
|
+
ICAPrb::Filters.get_logger.debug('Found match')
|
87
|
+
data[:http_response_body] = '<html><body>Blocked Hash</body></html>'
|
88
|
+
return true
|
89
|
+
end
|
90
|
+
end
|
91
|
+
false
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
# Check if the URL contains the given words
|
96
|
+
class URLContains
|
97
|
+
# Name in the configuration file
|
98
|
+
FILTER_NAME = 'url_contains'
|
99
|
+
# Available mod modes
|
100
|
+
MODES = [:response_mod]
|
101
|
+
|
102
|
+
# Constructor
|
103
|
+
# +mode+:: resp or req mod
|
104
|
+
# +parameters+:: All parameters given in the configuration file
|
105
|
+
def initialize(_, parameters)
|
106
|
+
@params = parameters
|
107
|
+
end
|
108
|
+
|
109
|
+
# Execute plugin
|
110
|
+
# +data+:: ICAP data
|
111
|
+
def plugin(data)
|
112
|
+
uri = data[:http_request_header]['Host']
|
113
|
+
@params.each { |url|
|
114
|
+
if uri.include? url
|
115
|
+
# Block by telling the user that this is blocked
|
116
|
+
data[:http_response_body] = '<html><body>Blocked URI</body></html>'
|
117
|
+
return true
|
118
|
+
end
|
119
|
+
}
|
120
|
+
false
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
# Check in headers for regex
|
125
|
+
class Headers
|
126
|
+
# Name in the configuration file
|
127
|
+
FILTER_NAME = 'headers'
|
128
|
+
# Available mod modes
|
129
|
+
MODES = [:response_mod]
|
130
|
+
# Constructor
|
131
|
+
# +mode+:: resp or req mod
|
132
|
+
# +parameters+:: All parameters given in the configuration file
|
133
|
+
def initialize(_, parameters)
|
134
|
+
@params = parameters
|
135
|
+
end
|
136
|
+
# Execute plugin
|
137
|
+
# +data+:: ICAP data
|
138
|
+
def plugin(data)
|
139
|
+
headers = data[:http_request_header]
|
140
|
+
headers.each { |header_name, header_content|
|
141
|
+
@params.each { |header_target, regex_data|
|
142
|
+
if header_name == header_target
|
143
|
+
regex_data.each{ |current_regex|
|
144
|
+
if header_content.match(current_regex)
|
145
|
+
data[:http_response_body] = '<html><body>Blocked header</body></html>'
|
146
|
+
return true
|
147
|
+
end
|
148
|
+
}
|
149
|
+
end
|
150
|
+
}
|
151
|
+
}
|
152
|
+
false
|
153
|
+
end
|
154
|
+
end
|
155
|
+
# Check if any attribute keys matching
|
156
|
+
class ContentHasKey
|
157
|
+
# Name in the configuration file
|
158
|
+
FILTER_NAME = 'content_has_key'
|
159
|
+
# Available mod modes
|
160
|
+
MODES = [:response_mod]
|
161
|
+
# Constructor
|
162
|
+
# +mode+:: resp or req mod
|
163
|
+
# +parameters+:: All parameters given in the configuration file
|
164
|
+
def initialize(_, parameters)
|
165
|
+
@params = parameters
|
166
|
+
end
|
167
|
+
# Execute plugin
|
168
|
+
# +data+:: ICAP data
|
169
|
+
def plugin(data)
|
170
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
171
|
+
doc.traverse do |node|
|
172
|
+
node.keys.each do |key|
|
173
|
+
@params.each do |attribute_key|
|
174
|
+
if key == attribute_key
|
175
|
+
data[:http_response_body] = '<html><body>Blocked attribute key</body></html>'
|
176
|
+
return true
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
180
|
+
end
|
181
|
+
false
|
182
|
+
end
|
183
|
+
end
|
184
|
+
# Check if the content of the tags includes any bad words
|
185
|
+
class ContentIncludes
|
186
|
+
# Name in the configuration file
|
187
|
+
FILTER_NAME = 'content_includes'
|
188
|
+
# Available mod modes
|
189
|
+
MODES = [:responese_mod]
|
190
|
+
# Constructor
|
191
|
+
# +mode+:: resp or req mod
|
192
|
+
# +parameters+:: All parameters given in the configuration file
|
193
|
+
def initialize(_, parameters)
|
194
|
+
@params = parameters
|
195
|
+
end
|
196
|
+
# Execute plugin
|
197
|
+
# +data+:: ICAP data
|
198
|
+
def plugin(data)
|
199
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
200
|
+
doc.traverse do |node|
|
201
|
+
@params.each { |phrase|
|
202
|
+
if node.text and node.text.include? phrase
|
203
|
+
data[:http_response_body] = '<html><body>Blocked content</body></html>'
|
204
|
+
return true
|
205
|
+
end
|
206
|
+
}
|
207
|
+
end
|
208
|
+
false
|
209
|
+
end
|
210
|
+
end
|
211
|
+
end
|
212
|
+
# Plugins for modify
|
213
|
+
module Plugins
|
214
|
+
# Send URLs to Virustotal and log into file
|
215
|
+
class VirusTotalURL
|
216
|
+
# Name in the configuration file
|
217
|
+
PLUGIN_NAME = 'send_url_to_virustotal'
|
218
|
+
# Available mod modes
|
219
|
+
MODES = [:request_mod]
|
220
|
+
|
221
|
+
# Constructor
|
222
|
+
# +mode+:: resp or req mod
|
223
|
+
# +parameters+:: All parameters given in the configuration file
|
224
|
+
def initialize(_, parameters)
|
225
|
+
@api_key = parameters[:api_key]
|
226
|
+
@log_file = File.expand_path(parameters[:log_file])
|
227
|
+
end
|
228
|
+
|
229
|
+
# Execute plugin
|
230
|
+
def plugin(data)
|
231
|
+
# Get address
|
232
|
+
uri = data[:http_request_header]['Host']
|
233
|
+
# Get result
|
234
|
+
ICAPrb::Filters.get_logger.debug("Sending uri #{uri.to_s} to Virustotal")
|
235
|
+
begin
|
236
|
+
results = Uirusu::VTUrl.query_report(@api_key, uri)
|
237
|
+
result = Uirusu::VTResult.new(uri, results)
|
238
|
+
# Write result
|
239
|
+
if result != nil
|
240
|
+
unless File.file?(@log_file)
|
241
|
+
FileUtils::touch @log_file
|
242
|
+
end
|
243
|
+
IO.write(@log_file, result.to_json.to_s.gsub('\n', '').gsub('\r', ''), mode: 'a')
|
244
|
+
end
|
245
|
+
rescue StandardError => e
|
246
|
+
ICAPrb::Filters.get_logger.warn('Virustotal error ' + e.message + e.backtrace.to_s)
|
247
|
+
end
|
248
|
+
false
|
249
|
+
end
|
250
|
+
end
|
251
|
+
# TODO at this time this should only work with request_mod
|
252
|
+
class ReplaceContent
|
253
|
+
# array of hashes in the form with the following fields
|
254
|
+
# match:: regular expression of url/
|
255
|
+
# content:: the content to use
|
256
|
+
# content_type:: 'set the content type'
|
257
|
+
attr_reader :replacement
|
258
|
+
# Constructor
|
259
|
+
# +mode+:: resp or req mod
|
260
|
+
# +parameters+:: All parameters given in the configuration file
|
261
|
+
def initialize(mode, parameters)
|
262
|
+
super
|
263
|
+
@replacement = []
|
264
|
+
end
|
265
|
+
|
266
|
+
# Name in the configuration file
|
267
|
+
PLUGIN_NAME = 'replace_content'
|
268
|
+
|
269
|
+
# Available mod modes
|
270
|
+
MODES = [:response_mod,:request_mod]
|
271
|
+
# Execute plugin
|
272
|
+
# +data+:: ICAP data
|
273
|
+
def plugin(data)
|
274
|
+
matching_replacements = @replacement.select {|replacement| data =~ replacement[:match] }
|
275
|
+
if matching_replacements.count > 0
|
276
|
+
matching_replacement = matching_replacements.first
|
277
|
+
data[:http_response_body] = ::ICAPrb::Server::ResponseBody.new(matching_replacement[:content])
|
278
|
+
if data[:http_response_header]
|
279
|
+
data[:http_response_header] =
|
280
|
+
::ICAPrb::Server::ResponseHeader.new(data[:http_response_header].http_version, 200)
|
281
|
+
elsif data[:http_request_header]
|
282
|
+
data[:http_response_header] =
|
283
|
+
::ICAPrb::Server::ResponseHeader.new(data[:http_request_header].http_version, 200)
|
284
|
+
else
|
285
|
+
data[:http_response_header] = ::ICAPrb::Server::ResponseHeader.new('1.1', 200)
|
286
|
+
end
|
287
|
+
|
288
|
+
data[:http_response_header]['Content-Type'] = matching_replacement[:content_type]
|
289
|
+
data[:http_response_header]['Content-Length'] = matching_replacement[:content].length
|
290
|
+
data.delete(:http_request_header) if data[:http_request_header]
|
291
|
+
return true
|
292
|
+
end
|
293
|
+
false
|
294
|
+
end
|
295
|
+
end
|
296
|
+
|
297
|
+
#This class is used to delete common implementations of facebook buttons
|
298
|
+
class RemoveFB
|
299
|
+
# +mode+:: resp or req mod
|
300
|
+
# +parameters+:: All parameters given in the configuration file
|
301
|
+
def initialize(_, _)
|
302
|
+
end
|
303
|
+
|
304
|
+
# class name
|
305
|
+
PLUGIN_NAME = 'remove_fb'
|
306
|
+
|
307
|
+
# Available mod modes
|
308
|
+
MODES = [:response_mod]
|
309
|
+
|
310
|
+
#execute the plug in
|
311
|
+
def plugin(data)
|
312
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
313
|
+
mod = false
|
314
|
+
|
315
|
+
# removes all iframes where the source matches
|
316
|
+
doc.search('iframe').each do |iframe|
|
317
|
+
if iframe['src'].match(/facebook\.com\/.*like\.php/i)
|
318
|
+
iframe.remove
|
319
|
+
mod = true
|
320
|
+
end
|
321
|
+
end
|
322
|
+
|
323
|
+
# removes all matching links
|
324
|
+
doc.search('a').each do |a|
|
325
|
+
if a['href'].to_s.match(/www\.facebook\.com\/sharer\.php|www\.facebook\.com\/sharer\/sharer\.php/)
|
326
|
+
a.remove
|
327
|
+
mod = true
|
328
|
+
end
|
329
|
+
end
|
330
|
+
|
331
|
+
# removes the facebook like-button element
|
332
|
+
doc.search('div.fb-like').each do |src|
|
333
|
+
src.remove
|
334
|
+
mod = true
|
335
|
+
end
|
336
|
+
|
337
|
+
# removes the facebook share-button element
|
338
|
+
doc.search('div.fb-share-button').each do |src|
|
339
|
+
src.remove
|
340
|
+
mod = true
|
341
|
+
end
|
342
|
+
|
343
|
+
# removes the facebook comments element
|
344
|
+
doc.search('div.fb-comments').each do |src|
|
345
|
+
src.remove
|
346
|
+
mod = true
|
347
|
+
end
|
348
|
+
|
349
|
+
# removes all scripts where the content matches
|
350
|
+
doc.css('script').each do |script|
|
351
|
+
if script.content.match(/connect\.facebook\.net\//i)
|
352
|
+
script.remove
|
353
|
+
mod = true
|
354
|
+
end
|
355
|
+
end
|
356
|
+
|
357
|
+
|
358
|
+
data[:http_response_body] = doc.to_s if mod
|
359
|
+
mod
|
360
|
+
end
|
361
|
+
end
|
362
|
+
|
363
|
+
#This class is used to delete common implementations of the Twitter-Share-Button
|
364
|
+
class RemoveRetweetButton
|
365
|
+
# +mode+:: resp or req mod
|
366
|
+
# +parameters+:: All parameters given in the configuration file
|
367
|
+
def initialize(_, _)
|
368
|
+
end
|
369
|
+
|
370
|
+
#class name
|
371
|
+
PLUGIN_NAME = 'remove_retweet_button'
|
372
|
+
|
373
|
+
# Available mod modes
|
374
|
+
MODES = [:response_mod]
|
375
|
+
# Execute plugin
|
376
|
+
# +data+:: ICAP data
|
377
|
+
def plugin(data)
|
378
|
+
mod = false
|
379
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
380
|
+
|
381
|
+
# removes twitter share button elements
|
382
|
+
doc.search('a.twitter-share-button').each do |src|
|
383
|
+
src.remove
|
384
|
+
mod = true
|
385
|
+
end
|
386
|
+
|
387
|
+
# removes twitter timeline elements
|
388
|
+
doc.search('a.twitter-timeline').each do |src|
|
389
|
+
src.remove
|
390
|
+
mod = true
|
391
|
+
end
|
392
|
+
|
393
|
+
# removes all matching links
|
394
|
+
doc.search('a').each do |a|
|
395
|
+
if a['href'].to_s.match(/twitter\.com\/share\?/i)
|
396
|
+
a.remove
|
397
|
+
mod = true
|
398
|
+
end
|
399
|
+
if a['href'].to_s.match(/twitter\.com\/intent\//i)
|
400
|
+
a.remove
|
401
|
+
mod = true
|
402
|
+
end
|
403
|
+
end
|
404
|
+
|
405
|
+
# removes all scripts where the content matches
|
406
|
+
doc.css('script').each do |script|
|
407
|
+
if script.content.match(/\/\/platform\.twitter\.com\/widgets\.js/i)
|
408
|
+
script.remove
|
409
|
+
mod = true
|
410
|
+
end
|
411
|
+
end
|
412
|
+
|
413
|
+
# removes all iframes where the source matches
|
414
|
+
doc.search('iframe').each do |iframe|
|
415
|
+
if iframe['src'].match(/platform\.twitter\.com\/widgets/i)
|
416
|
+
iframe.remove
|
417
|
+
mod = true
|
418
|
+
end
|
419
|
+
end
|
420
|
+
|
421
|
+
data[:http_response_body] = doc.to_s if mod
|
422
|
+
mod
|
423
|
+
end
|
424
|
+
end
|
425
|
+
|
426
|
+
#Plug in to remove the Google PlusOne Button
|
427
|
+
class RemovePlusOneButton
|
428
|
+
# +mode+:: resp or req mod
|
429
|
+
# +parameters+:: All parameters given in the configuration file
|
430
|
+
def initialize(_, _)
|
431
|
+
end
|
432
|
+
|
433
|
+
# class name
|
434
|
+
PLUGIN_NAME = 'remove_plusone_button'
|
435
|
+
|
436
|
+
# Available mod modes
|
437
|
+
MODES = [:response_mod]
|
438
|
+
|
439
|
+
# execute the plug in
|
440
|
+
def plugin(data)
|
441
|
+
mod = false
|
442
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
443
|
+
|
444
|
+
# removes all g-plusone elements
|
445
|
+
doc.search('div.g-plusone').each do |src|
|
446
|
+
src.remove
|
447
|
+
mod = true
|
448
|
+
end
|
449
|
+
|
450
|
+
# removes all matching links
|
451
|
+
doc.search('a').each do |a|
|
452
|
+
if a['href'].to_s.match(/plus\.google\.com\/share\?/i)
|
453
|
+
end
|
454
|
+
end
|
455
|
+
|
456
|
+
# removes all scripts where the source or the content matches
|
457
|
+
doc.css('script').each do |script|
|
458
|
+
if script['src'] == 'https://apis.google.com/js/platform.js' || script.content.match(/apis\.google\.com/i)
|
459
|
+
script.remove
|
460
|
+
mod = true
|
461
|
+
end
|
462
|
+
end
|
463
|
+
|
464
|
+
# removes all iframes where the source matches
|
465
|
+
doc.search('iframe').each do |iframe|
|
466
|
+
if iframe['src'].match(/apis\.google\.com/i)
|
467
|
+
iframe.remove
|
468
|
+
mod = true
|
469
|
+
end
|
470
|
+
end
|
471
|
+
|
472
|
+
data[:http_response_body] = doc.to_s if mod
|
473
|
+
mod
|
474
|
+
end
|
475
|
+
end
|
476
|
+
|
477
|
+
# remove the two click solution provided by heise.de
|
478
|
+
class RemoveSocialSharePrivacy
|
479
|
+
# +mode+:: resp or req mod
|
480
|
+
# +parameters+:: All parameters given in the configuration file
|
481
|
+
def initialize(_, _)
|
482
|
+
end
|
483
|
+
|
484
|
+
# class name
|
485
|
+
PLUGIN_NAME = 'remove_social_share_privacy'
|
486
|
+
|
487
|
+
# Available mod modes
|
488
|
+
MODES = [:response_mod]
|
489
|
+
|
490
|
+
# execute the plug in
|
491
|
+
def plugin(data)
|
492
|
+
mod = false
|
493
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
494
|
+
|
495
|
+
# removes all socialshareprivacy elements
|
496
|
+
doc.css('#socialshareprivacy').each do |src|
|
497
|
+
src.remove
|
498
|
+
mod = true
|
499
|
+
end
|
500
|
+
|
501
|
+
# removes all scripts where the source or the content matches
|
502
|
+
doc.css('script').each do |script|
|
503
|
+
if script.content.match(/socialshareprivacy\.css/i) || (script['src'] == 'jquery.socialshareprivacy.js')
|
504
|
+
script.remove
|
505
|
+
mod = true
|
506
|
+
end
|
507
|
+
end
|
508
|
+
data[:http_response_body] = doc.to_s if mod
|
509
|
+
mod
|
510
|
+
end
|
511
|
+
end
|
512
|
+
|
513
|
+
#Plug in to remove all Google AdWords
|
514
|
+
class RemoveGoogleAdWords
|
515
|
+
# +mode+:: resp or req mod
|
516
|
+
# +parameters+:: All parameters given in the configuration file
|
517
|
+
def initialize(_, _)
|
518
|
+
end
|
519
|
+
|
520
|
+
# class name
|
521
|
+
PLUGIN_NAME = 'remove_google_ad_words'
|
522
|
+
|
523
|
+
# Available mod modes
|
524
|
+
MODES = [:response_mod]
|
525
|
+
|
526
|
+
# execute the plug in
|
527
|
+
# google AdWords are loaded with JSON files
|
528
|
+
# the plugin looks for the source record of google Adwords in those
|
529
|
+
# files and deletes them
|
530
|
+
def plugin(data)
|
531
|
+
mod = false
|
532
|
+
joined = ''
|
533
|
+
|
534
|
+
if data[:icap_data][:request_line][:uri].to_s.match(/www\.google\..*/i)
|
535
|
+
if data[:http_response_header]['Content-Type']=='application/json'
|
536
|
+
splitted = data[:http_response_body].split('/*""*/')
|
537
|
+
parsed = splitted.map {|x| JSON.parse x if x.length > 2}
|
538
|
+
parsed.each do |x|
|
539
|
+
if x && ( x['d'].include? 'www.googleadservices.com')
|
540
|
+
x['d'] = ''
|
541
|
+
mod = true
|
542
|
+
end
|
543
|
+
end
|
544
|
+
if mod
|
545
|
+
parsed.each do |x|
|
546
|
+
joined = joined + x.to_json + '/*""*/'
|
547
|
+
end
|
548
|
+
data[:http_response_body] = joined
|
549
|
+
end
|
550
|
+
end
|
551
|
+
end
|
552
|
+
mod
|
553
|
+
end
|
554
|
+
end
|
555
|
+
|
556
|
+
# Plug in to remove the Google Webads
|
557
|
+
class RemoveGoogleWebAds
|
558
|
+
# +mode+:: resp or req mod
|
559
|
+
# +parameters+:: All parameters given in the configuration file
|
560
|
+
def initialize(_, _)
|
561
|
+
end
|
562
|
+
|
563
|
+
# class name
|
564
|
+
PLUGIN_NAME = 'remove_google_web_ads'
|
565
|
+
|
566
|
+
# Available mod modes
|
567
|
+
MODES = [:response_mod]
|
568
|
+
|
569
|
+
# execute the plug in
|
570
|
+
def plugin(data)
|
571
|
+
mod = false
|
572
|
+
|
573
|
+
#stops loading from specific hosts
|
574
|
+
if data[:http_request_header]['Host'].to_s.match(/googleads|googlesyndicat|googleadservice|\.doubleclick\./i)
|
575
|
+
data[:http_response_body] = ''
|
576
|
+
mod = true
|
577
|
+
end
|
578
|
+
|
579
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
580
|
+
|
581
|
+
#ICAPrb::Filters.get_logger.debug('content: ' + content = data[:http_response_body].to_s)
|
582
|
+
|
583
|
+
#delete divs where div.class contains keyword
|
584
|
+
doc.search('div').each do |src|
|
585
|
+
if src['class'].to_s.match(/ads|google_ads|google_companion|google-companion|googlesyndicat/i) || src['id'].to_s.match(/google_ads/i)
|
586
|
+
src.remove
|
587
|
+
mod = true
|
588
|
+
end
|
589
|
+
end
|
590
|
+
|
591
|
+
#delete hyperlinks where href contains keyword
|
592
|
+
doc.search('a').each do |a|
|
593
|
+
if a['href'].to_s.match(/googleads|googleadservice|googlesyndicat|\.doubleclick\./i)
|
594
|
+
a.remove
|
595
|
+
mod = true
|
596
|
+
end
|
597
|
+
end
|
598
|
+
|
599
|
+
#delete scripts where src or content matches keyword
|
600
|
+
doc.search('script').each do |script|
|
601
|
+
if script.content.match(/googleads|.*googlesyndicat.*|googleadservice|\.doubleclick\./i) || script['src'].to_s.match(/googleads|googlesyndicat|googleadservice|\.doubleclick\./i)
|
602
|
+
script.remove
|
603
|
+
mod = true
|
604
|
+
end
|
605
|
+
end
|
606
|
+
|
607
|
+
#delete images where the source matches
|
608
|
+
doc.search('img').each do |img|
|
609
|
+
if img['src'].to_s.match(/googleads\.|\.doubleclick\./i)
|
610
|
+
img.remove
|
611
|
+
mod = true
|
612
|
+
end
|
613
|
+
end
|
614
|
+
|
615
|
+
# removes google Ad Elements
|
616
|
+
doc.search('gwd-doubleclick').each do |gwd|
|
617
|
+
gwd.remove
|
618
|
+
mod = true
|
619
|
+
end
|
620
|
+
|
621
|
+
# removes google Ad Elements
|
622
|
+
doc.search('gwd-genericad').each do |gwd|
|
623
|
+
gwd.remove
|
624
|
+
mod = true
|
625
|
+
end
|
626
|
+
|
627
|
+
data[:http_response_body] = doc.to_s if mod
|
628
|
+
|
629
|
+
mod
|
630
|
+
end
|
631
|
+
end
|
632
|
+
|
633
|
+
# Plug in to remove the Adition Webads
|
634
|
+
class RemoveAditionWebAds
|
635
|
+
# +mode+:: resp or req mod
|
636
|
+
# +parameters+:: All parameters given in the configuration file
|
637
|
+
def initialize(_, _)
|
638
|
+
end
|
639
|
+
|
640
|
+
# class name
|
641
|
+
PLUGIN_NAME = 'remove_adition_web_ads'
|
642
|
+
|
643
|
+
# Available mod modes
|
644
|
+
MODES = [:response_mod]
|
645
|
+
|
646
|
+
# execute the plug in
|
647
|
+
# removes all links and scripts where the source or the content matches
|
648
|
+
def plugin(data)
|
649
|
+
mod = false
|
650
|
+
|
651
|
+
#stops loading from specific hosts
|
652
|
+
if data[:http_request_header]['Host'].to_s.match(/\.adfarm[0-9]+\.adition/i)
|
653
|
+
data[:http_response_body] = ''
|
654
|
+
mod = true
|
655
|
+
end
|
656
|
+
|
657
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
658
|
+
|
659
|
+
doc.search('a').each do |a|
|
660
|
+
if a['href'].to_s.match(/\.adfarm[0-9]+\.adition\.com/i)
|
661
|
+
a.remove
|
662
|
+
mod = true
|
663
|
+
end
|
664
|
+
end
|
665
|
+
|
666
|
+
doc.search('script').each do |script|
|
667
|
+
if script['src'].to_s.match(/\.adfarm[0-9]+\.adition/i) || script.content.to_s.match(/adition|imagesrv\.adition\.com\/js\/srp\.js/i)
|
668
|
+
script.remove
|
669
|
+
mod = true
|
670
|
+
end
|
671
|
+
end
|
672
|
+
data[:http_response_body] = doc.to_s if mod
|
673
|
+
mod
|
674
|
+
end
|
675
|
+
end
|
676
|
+
|
677
|
+
# Plug in to remove the Nuggad Webads
|
678
|
+
class RemoveNuggadWebAds
|
679
|
+
# +mode+:: resp or req mod
|
680
|
+
# +parameters+:: All parameters given in the configuration file
|
681
|
+
def initialize(_, _)
|
682
|
+
end
|
683
|
+
|
684
|
+
# class name
|
685
|
+
PLUGIN_NAME = 'remove_nuggad_web_ads'
|
686
|
+
|
687
|
+
# Available mod modes
|
688
|
+
MODES = [:response_mod]
|
689
|
+
|
690
|
+
# execute the plug in
|
691
|
+
# removes all scripts where the source or the content matches
|
692
|
+
def plugin(data)
|
693
|
+
mod = false
|
694
|
+
|
695
|
+
#stops loading from specific host
|
696
|
+
if data[:http_request_header]['Host'].to_s.match(/\.nuggad\.net/i)
|
697
|
+
data[:http_response_body] = ''
|
698
|
+
mod = true
|
699
|
+
end
|
700
|
+
|
701
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
702
|
+
|
703
|
+
doc.search('script').each do |script|
|
704
|
+
if script['src'].to_s.match(/\.nuggad\.net/i) || script.content.match(/\.nuggad\.net/i)
|
705
|
+
script.remove
|
706
|
+
mod = true
|
707
|
+
end
|
708
|
+
end
|
709
|
+
data[:http_response_body] = doc.to_s if mod
|
710
|
+
mod
|
711
|
+
end
|
712
|
+
end
|
713
|
+
|
714
|
+
# Plug in to remove the Google Tag Manager
|
715
|
+
class RemoveGoogleTagManager
|
716
|
+
# +mode+:: resp or req mod
|
717
|
+
# +parameters+:: All parameters given in the configuration file
|
718
|
+
def initialize(_, _)
|
719
|
+
end
|
720
|
+
|
721
|
+
# class name
|
722
|
+
PLUGIN_NAME = 'remove_google_tag_manager'
|
723
|
+
|
724
|
+
# Available mod modes
|
725
|
+
MODES = [:response_mod]
|
726
|
+
|
727
|
+
# execute the plug in
|
728
|
+
# removes all scripts where the source or the content matches
|
729
|
+
def plugin(data)
|
730
|
+
mod = false
|
731
|
+
|
732
|
+
#stops loading from specific hosts
|
733
|
+
if data[:http_request_header]['Host'].to_s.match(/googletagservices\.com/i)
|
734
|
+
data[:http_response_body] = ''
|
735
|
+
mod = true
|
736
|
+
end
|
737
|
+
|
738
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
739
|
+
|
740
|
+
doc.search('script').each do |script|
|
741
|
+
if script['src'].to_s.match(/www\.googletagservices\.com\/tag\/js\/gpt\.js/i) || script.content.match(/googletagservices|googletag/i)
|
742
|
+
script.remove
|
743
|
+
mod = true
|
744
|
+
end
|
745
|
+
end
|
746
|
+
data[:http_response_body] = doc.to_s if mod
|
747
|
+
mod
|
748
|
+
end
|
749
|
+
end
|
750
|
+
|
751
|
+
# Plug in to remove OEWA tracker.
|
752
|
+
class RemoveOewaTracker
|
753
|
+
# +mode+:: resp or req mod
|
754
|
+
# +parameters+:: All parameters given in the configuration file
|
755
|
+
def initialize(_, _)
|
756
|
+
end
|
757
|
+
|
758
|
+
# class name
|
759
|
+
PLUGIN_NAME = 'remove_oewa_tracker'
|
760
|
+
|
761
|
+
# Available mod modes
|
762
|
+
MODES = [:response_mod]
|
763
|
+
|
764
|
+
# execute the plug in
|
765
|
+
# removes all scripts where the source or the content matches
|
766
|
+
def plugin(data)
|
767
|
+
mod = false
|
768
|
+
|
769
|
+
#stops loading from specific host
|
770
|
+
if data[:http_request_header]['Host'].to_s.match(/oewabox\.at/i)
|
771
|
+
data[:http_response_body] = ''
|
772
|
+
mod = true
|
773
|
+
end
|
774
|
+
|
775
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
776
|
+
|
777
|
+
doc.search('script').each do |script|
|
778
|
+
if script['src'].to_s.match(/oewabox\.at\/oewa\.js/i) || script.content.match(/oewabox\.at\/oewa\.js/i)
|
779
|
+
script.remove
|
780
|
+
mod = true
|
781
|
+
end
|
782
|
+
end
|
783
|
+
data[:http_response_body] = doc.to_s if mod
|
784
|
+
mod
|
785
|
+
end
|
786
|
+
end
|
787
|
+
|
788
|
+
# Plug in to remove the Piwik Tracker
|
789
|
+
class RemovePiwikTracker
|
790
|
+
# +mode+:: resp or req mod
|
791
|
+
# +parameters+:: All parameters given in the configuration file
|
792
|
+
def initialize(_, _)
|
793
|
+
end
|
794
|
+
|
795
|
+
# class name
|
796
|
+
PLUGIN_NAME = 'remove_piwik_tracker'
|
797
|
+
|
798
|
+
# Available mod modes
|
799
|
+
MODES = [:response_mod]
|
800
|
+
|
801
|
+
# execute the plug in
|
802
|
+
# removes all scripts where the source or the content matches
|
803
|
+
def plugin(data)
|
804
|
+
mod = false
|
805
|
+
|
806
|
+
#stops loading from specific host
|
807
|
+
if data[:http_request_header]['Host'].to_s.match(/demo[0-9]+\.piwik\.org/i)
|
808
|
+
data[:http_response_body] = ''
|
809
|
+
mod = true
|
810
|
+
end
|
811
|
+
|
812
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
813
|
+
|
814
|
+
doc.search('script').each do |script|
|
815
|
+
if script['src'].to_s.match(/piwik\.js/i) || script.content.match(/piwik\.js/i)
|
816
|
+
script.remove
|
817
|
+
mod = true
|
818
|
+
end
|
819
|
+
end
|
820
|
+
data[:http_response_body] = doc.to_s if mod
|
821
|
+
mod
|
822
|
+
end
|
823
|
+
end
|
824
|
+
|
825
|
+
# Plug in to remove the Google Analytics tracker
|
826
|
+
class RemoveGoogleAnaytics
|
827
|
+
# +mode+:: resp or req mod
|
828
|
+
# +parameters+:: All parameters given in the configuration file
|
829
|
+
def initialize(_, _)
|
830
|
+
end
|
831
|
+
|
832
|
+
# class name
|
833
|
+
PLUGIN_NAME = 'remove_google_analytics'
|
834
|
+
|
835
|
+
# Available mod modes
|
836
|
+
MODES = [:response_mod]
|
837
|
+
|
838
|
+
# execute the plug in
|
839
|
+
# removes all scripts where the source or the content matches
|
840
|
+
def plugin(data)
|
841
|
+
mod = false
|
842
|
+
|
843
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
844
|
+
|
845
|
+
doc.search('script').each do |script|
|
846
|
+
if script['src'].to_s.match(/\/analytics\.js/i) || script.content.match(/\/analytics\.js/i)
|
847
|
+
script.remove
|
848
|
+
mod = true
|
849
|
+
end
|
850
|
+
end
|
851
|
+
data[:http_response_body] = doc.to_s if mod
|
852
|
+
mod
|
853
|
+
end
|
854
|
+
end
|
855
|
+
|
856
|
+
# Plug in to remove the ioam.de tracker.
|
857
|
+
class RemoveIoam
|
858
|
+
# +mode+:: resp or req mod
|
859
|
+
# +parameters+:: All parameters given in the configuration file
|
860
|
+
def initialize(_, _)
|
861
|
+
end
|
862
|
+
|
863
|
+
# class name
|
864
|
+
PLUGIN_NAME = 'remove_ioam'
|
865
|
+
|
866
|
+
# Available mod modes
|
867
|
+
MODES = [:response_mod]
|
868
|
+
|
869
|
+
# execute the plug in
|
870
|
+
# removes all scripts where the source or the content matches
|
871
|
+
def plugin(data)
|
872
|
+
mod = false
|
873
|
+
|
874
|
+
#stops loading from specific host
|
875
|
+
if data[:http_request_header]['Host'].to_s.match(/script\.ioam.de/i)
|
876
|
+
data[:http_response_body] = ''
|
877
|
+
mod = true
|
878
|
+
end
|
879
|
+
|
880
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
881
|
+
|
882
|
+
doc.search('script').each do |script|
|
883
|
+
if script['src'].to_s.match(/script\.ioam.de\/iam\.js/i) || script.content.match(/ioam.de\/iam\.js/i)
|
884
|
+
script.remove
|
885
|
+
mod = true
|
886
|
+
end
|
887
|
+
end
|
888
|
+
data[:http_response_body] = doc.to_s if mod
|
889
|
+
mod
|
890
|
+
end
|
891
|
+
end
|
892
|
+
|
893
|
+
# Plug in to remove the Webtrekk Tracker
|
894
|
+
class RemoveWebtrekk# Constructor
|
895
|
+
# +mode+:: resp or req mod
|
896
|
+
# +parameters+:: All parameters given in the configuration file
|
897
|
+
def initialize(_, _)
|
898
|
+
end
|
899
|
+
|
900
|
+
# class name
|
901
|
+
PLUGIN_NAME = 'remove_webtrekk'
|
902
|
+
|
903
|
+
# Available mod modes
|
904
|
+
MODES = [:response_mod]
|
905
|
+
|
906
|
+
# execute the plug in
|
907
|
+
# removes all scripts where the source or the content matches
|
908
|
+
def plugin(data)
|
909
|
+
mod = false
|
910
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
911
|
+
|
912
|
+
doc.search('script').each do |script|
|
913
|
+
if script['src'].to_s.match(/webtrekk[\S]*\.js/i) || script.content.match(/webtrekk[\S]*\.js/i)
|
914
|
+
script.remove
|
915
|
+
mod = true
|
916
|
+
end
|
917
|
+
end
|
918
|
+
data[:http_response_body] = doc.to_s if mod
|
919
|
+
mod
|
920
|
+
end
|
921
|
+
end
|
922
|
+
|
923
|
+
# Plug in to remove the Visual Revenue Tracker
|
924
|
+
class RemoveVisualRevenue
|
925
|
+
# Constructor
|
926
|
+
# +mode+:: resp or req mod
|
927
|
+
# +parameters+:: All parameters given in the configuration file
|
928
|
+
def initialize(_, _)
|
929
|
+
end
|
930
|
+
|
931
|
+
# class name
|
932
|
+
PLUGIN_NAME = 'remove_visual_revenue'
|
933
|
+
|
934
|
+
# Available mod modes
|
935
|
+
MODES = [:response_mod]
|
936
|
+
|
937
|
+
# execute the plug in
|
938
|
+
# removes all scripts where the source or the content matches
|
939
|
+
def plugin(data)
|
940
|
+
mod = false
|
941
|
+
doc = Nokogiri::HTML(data[:http_response_body])
|
942
|
+
|
943
|
+
doc.search('script').each do |script|
|
944
|
+
if script['src'].to_s.match(/visualrevenue\.com\/vrs\.js/i) || script.content.match(/visualrevenue\.com\/vrs\.js/i)
|
945
|
+
script.remove
|
946
|
+
mod = true
|
947
|
+
end
|
948
|
+
end
|
949
|
+
data[:http_response_body] = doc.to_s if mod
|
950
|
+
mod
|
951
|
+
end
|
952
|
+
end
|
953
|
+
end
|
954
|
+
end
|