exception_handling 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +1 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +79 -0
- data/LICENSE +22 -0
- data/README +1 -0
- data/Rakefile +9 -0
- data/config/exception_filters.yml +138 -0
- data/exception_handling.gemspec +24 -0
- data/lib/exception_handling/version.rb +3 -0
- data/lib/exception_handling.rb +630 -0
- data/lib/exception_handling_mailer.rb +77 -0
- data/test/exception_handling_test.rb +923 -0
- data/test/mocha_patch.rb +63 -0
- data/test/test_helper.rb +93 -0
- data/views/exception_handling/mailer/escalation_notification.html.erb +14 -0
- data/views/exception_handling/mailer/exception_notification.html.erb +81 -0
- data/views/exception_handling/mailer/log_parser_exception_notification.html.erb +82 -0
- metadata +177 -0
data/.gitignore
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
.idea
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,79 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
exception_handling (0.1.1)
|
5
|
+
actionmailer (= 3.2.13)
|
6
|
+
actionpack (= 3.2.13)
|
7
|
+
activesupport (= 3.2.13)
|
8
|
+
eventmachine (>= 0.12.10)
|
9
|
+
|
10
|
+
GEM
|
11
|
+
remote: https://rubygems.org/
|
12
|
+
specs:
|
13
|
+
actionmailer (3.2.13)
|
14
|
+
actionpack (= 3.2.13)
|
15
|
+
mail (~> 2.5.3)
|
16
|
+
actionpack (3.2.13)
|
17
|
+
activemodel (= 3.2.13)
|
18
|
+
activesupport (= 3.2.13)
|
19
|
+
builder (~> 3.0.0)
|
20
|
+
erubis (~> 2.7.0)
|
21
|
+
journey (~> 1.0.4)
|
22
|
+
rack (~> 1.4.5)
|
23
|
+
rack-cache (~> 1.2)
|
24
|
+
rack-test (~> 0.6.1)
|
25
|
+
sprockets (~> 2.2.1)
|
26
|
+
activemodel (3.2.13)
|
27
|
+
activesupport (= 3.2.13)
|
28
|
+
builder (~> 3.0.0)
|
29
|
+
activesupport (3.2.13)
|
30
|
+
i18n (= 0.6.1)
|
31
|
+
multi_json (~> 1.0)
|
32
|
+
bourne (1.3.0)
|
33
|
+
mocha (= 0.13.0)
|
34
|
+
builder (3.0.4)
|
35
|
+
erubis (2.7.0)
|
36
|
+
eventmachine (1.0.3)
|
37
|
+
hike (1.2.3)
|
38
|
+
i18n (0.6.1)
|
39
|
+
journey (1.0.4)
|
40
|
+
mail (2.5.4)
|
41
|
+
mime-types (~> 1.16)
|
42
|
+
treetop (~> 1.4.8)
|
43
|
+
metaclass (0.0.1)
|
44
|
+
mime-types (1.24)
|
45
|
+
mocha (0.13.0)
|
46
|
+
metaclass (~> 0.0.1)
|
47
|
+
multi_json (1.7.8)
|
48
|
+
polyglot (0.3.3)
|
49
|
+
rack (1.4.5)
|
50
|
+
rack-cache (1.2)
|
51
|
+
rack (>= 0.4)
|
52
|
+
rack-test (0.6.2)
|
53
|
+
rack (>= 1.0)
|
54
|
+
rake (10.1.0)
|
55
|
+
shoulda (3.1.1)
|
56
|
+
shoulda-context (~> 1.0)
|
57
|
+
shoulda-matchers (~> 1.2)
|
58
|
+
shoulda-context (1.1.4)
|
59
|
+
shoulda-matchers (1.5.6)
|
60
|
+
activesupport (>= 3.0.0)
|
61
|
+
bourne (~> 1.3)
|
62
|
+
sprockets (2.2.2)
|
63
|
+
hike (~> 1.2)
|
64
|
+
multi_json (~> 1.0)
|
65
|
+
rack (~> 1.0)
|
66
|
+
tilt (~> 1.1, != 1.3.0)
|
67
|
+
tilt (1.4.1)
|
68
|
+
treetop (1.4.15)
|
69
|
+
polyglot
|
70
|
+
polyglot (>= 0.3.1)
|
71
|
+
|
72
|
+
PLATFORMS
|
73
|
+
ruby
|
74
|
+
|
75
|
+
DEPENDENCIES
|
76
|
+
exception_handling!
|
77
|
+
mocha (= 0.13.0)
|
78
|
+
rake (>= 0.9)
|
79
|
+
shoulda (= 3.1.1)
|
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2012 Colin Kelley, RingRevenue
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
exception_handling gem
|
data/Rakefile
ADDED
@@ -0,0 +1,138 @@
|
|
1
|
+
--- !map:HashWithIndifferentAccess
|
2
|
+
All script kiddies:
|
3
|
+
error: "ScriptKiddie suspected because of HTTP request without a referer"
|
4
|
+
|
5
|
+
Tony Robbins:
|
6
|
+
request: " url: https?\:\/\/www\.tonyrobbins\.com"
|
7
|
+
|
8
|
+
Soulful Beauty:
|
9
|
+
request: "HTTP_REFERER: http://.*soulfulbeauty\.com"
|
10
|
+
|
11
|
+
The Credit Exchange:
|
12
|
+
error: "VirtualLine.find_by_external_ids - No promo number found.*Affiliate id from network (1389811|3079560|3366822|3179522|3375281|3375756|3362634|550151) not found"
|
13
|
+
request: "av_id: 178$"
|
14
|
+
|
15
|
+
Spiders:
|
16
|
+
environment: "Microsoft-WebDAV|COMODOspider|Baiduspider|YandexBot|TurnitinBot"
|
17
|
+
|
18
|
+
Googlebot:
|
19
|
+
environment: "googlebot"
|
20
|
+
error: "ReferenceError: Can't find variable: RingRevenue"
|
21
|
+
|
22
|
+
Options Request:
|
23
|
+
environment: "REQUEST_METHOD: OPTIONS"
|
24
|
+
|
25
|
+
Click Request Rejected:
|
26
|
+
error: "Request to click domain rejected"
|
27
|
+
request: "controller: corporate"
|
28
|
+
|
29
|
+
Staging Full Started:
|
30
|
+
error: "Mysql::Error: Can't connect to MySQL server on 'stagingmaster.ringrevenue.com'"
|
31
|
+
|
32
|
+
Google Analytics:
|
33
|
+
error: "ActionController.*RoutingError*google-analytics.*"
|
34
|
+
|
35
|
+
TermsLinksBroken:
|
36
|
+
error: "Logged in user experienced broken link on.*advertiser_campaign_terms"
|
37
|
+
|
38
|
+
NoRouteOnTerms:
|
39
|
+
error: "ActionController.*RoutingError.*No route matches.*campaign_terms"
|
40
|
+
|
41
|
+
InvalidAuth:
|
42
|
+
error: "Invalid authenticity token received from.*new\?"
|
43
|
+
|
44
|
+
NoRoute:
|
45
|
+
error: "No route matches"
|
46
|
+
|
47
|
+
SearchWebPlacementsJson:
|
48
|
+
error: "search_web_placement Invalid JSON response from yahoo api"
|
49
|
+
|
50
|
+
Rss 404s from Java:
|
51
|
+
request: "controller: articles"
|
52
|
+
error: "ActionController.*UnknownAction"
|
53
|
+
environment: "HTTP_USER_AGENT: Java"
|
54
|
+
|
55
|
+
Archiver mangling caused missing template:
|
56
|
+
error: "Missing template corporate/platform.erb"
|
57
|
+
environment: "(aihit|archive)"
|
58
|
+
|
59
|
+
Number Loader 404s from invalid url concatenation:
|
60
|
+
request: "url: https?:\/\/js\d{0,2}\.ringrevenue.com\/"
|
61
|
+
error: "Broken link.*ActiveRecord::RecordNotFound"
|
62
|
+
|
63
|
+
collegequest bad TLD:
|
64
|
+
error: "Hostname 'me.cq' does not have a known public suffix"
|
65
|
+
|
66
|
+
Ignite calls API:
|
67
|
+
request: "calls\/10\.xml"
|
68
|
+
|
69
|
+
BrokenLinkAfterLogin:
|
70
|
+
error: "Found broken link after user logged in from"
|
71
|
+
|
72
|
+
Link Trust Pixel Firing:
|
73
|
+
error: "Errno::ECONNREFUSED.*(linktrust\.com|mobitracking\.com|xy7track\.com|mbltrack\.com)"
|
74
|
+
|
75
|
+
# JAVASCRIPT exceptions
|
76
|
+
# http://www.roslindesign.com/2010/10/12/avg-antivirus-2011-corrupting-web-pages-with-injection-of-script-avg_ls_dom-js/
|
77
|
+
AVG Anti Virus:
|
78
|
+
request: "avg_ls_dom\.js"
|
79
|
+
|
80
|
+
Firefox Framework Bug:
|
81
|
+
error: "Javascript: window.onerror: Permission denied to access property 'nodeType' from a non-chrome context"
|
82
|
+
request: "action: javascript_error"
|
83
|
+
|
84
|
+
Javascript Error Loading Script:
|
85
|
+
error: "Javascript: window.onerror: Error loading script"
|
86
|
+
request: "filename: .*hubspot\.com.*"
|
87
|
+
|
88
|
+
Javascript Plugin Errors:
|
89
|
+
error: "Javascript: window.onerror: Script error"
|
90
|
+
request: "filename: chrome://leapfrog.*"
|
91
|
+
|
92
|
+
Javascript users with browser plugin issues:
|
93
|
+
error: "Javascript: .*"
|
94
|
+
session: "user_id: (42595|58296|61276|74059)"
|
95
|
+
|
96
|
+
Calls API Errors:
|
97
|
+
error: "calls_api returning status not_found"
|
98
|
+
|
99
|
+
Duplicate Sales Reported:
|
100
|
+
error: "Call.update_order called with duplicate sales"
|
101
|
+
|
102
|
+
Sale for the same reason and sku:
|
103
|
+
error: "Sale for the same reason and sku"
|
104
|
+
|
105
|
+
Found Inconsistent Caller Id Settings:
|
106
|
+
error: "CampaignIds:CampaignTerms_7542,CampaignTerms_7544,CampaignTerms_8242,CampaignTerms_8478,CampaignTerms_9244"
|
107
|
+
|
108
|
+
Invalid tacked_action referrer:
|
109
|
+
error: "StandardError: process_click_from_network_id - saving click failed.: Mysql::Error: Incorrect string value:.*for column 'referrer'.*INSERT INTO `tracked_actions`"
|
110
|
+
|
111
|
+
invalid click domain rdparking:
|
112
|
+
error: "Request to click domain rejected"
|
113
|
+
request: ".*rdparking.*"
|
114
|
+
|
115
|
+
update order with duplicate sales:
|
116
|
+
error: "update_order called with duplicate sales information"
|
117
|
+
|
118
|
+
failsafe on vxml because readonly:
|
119
|
+
error: "The MySQL server is running with the --read-only option so it cannot execute this statement.*INSERT INTO simple_sessions"
|
120
|
+
|
121
|
+
failsafe on propfind:
|
122
|
+
error: "UnknownHttpMethod: PROPFIND"
|
123
|
+
|
124
|
+
pixel_misconfigured:
|
125
|
+
error: "PixelUrlTemplate substitution"
|
126
|
+
|
127
|
+
buffered_sale:
|
128
|
+
error: "Couldn't apply BufferedSale"
|
129
|
+
|
130
|
+
SQL Injection Attempts:
|
131
|
+
request: "controller: virtual_lines"
|
132
|
+
request: "av_id: \D\d*\D\d*\D\d*\D"
|
133
|
+
|
134
|
+
Opera Browser:
|
135
|
+
environment: "HTTP_USER_AGENT: Opera"
|
136
|
+
|
137
|
+
Affiliate Map Number invalid requests:
|
138
|
+
error: "Publisher Map Number returning with invalid campaign"
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require File.expand_path('../lib/exception_handling/version', __FILE__)
|
2
|
+
|
3
|
+
Gem::Specification.new do |gem|
|
4
|
+
gem.add_dependency 'eventmachine', '>=0.12.10'
|
5
|
+
gem.add_dependency 'activesupport', '3.2.13'
|
6
|
+
gem.add_dependency 'actionpack', '3.2.13'
|
7
|
+
gem.add_dependency 'actionmailer', '3.2.13'
|
8
|
+
gem.add_development_dependency 'rake', '>=0.9'
|
9
|
+
gem.add_development_dependency 'shoulda', '=3.1.1'
|
10
|
+
gem.add_development_dependency 'mocha', '=0.13.0'
|
11
|
+
|
12
|
+
gem.authors = ["Colin Kelley"]
|
13
|
+
gem.email = ["colindkelley@gmail.com"]
|
14
|
+
gem.description = %q{Exception handling logger/emailer}
|
15
|
+
gem.summary = %q{RingRevenue's exception handling logger/emailer layer, based on exception_notifier. Works with Rails or EventMachine or EventMachine+Synchrony.}
|
16
|
+
gem.homepage = "https://github.com/RingRevenue/exception_handling"
|
17
|
+
|
18
|
+
gem.files = `git ls-files`.split($\)
|
19
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
20
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/.*\.rb})
|
21
|
+
gem.name = "exception_handling"
|
22
|
+
gem.require_paths = ["lib"]
|
23
|
+
gem.version = ExceptionHandling::VERSION
|
24
|
+
end
|
@@ -0,0 +1,630 @@
|
|
1
|
+
require 'timeout'
|
2
|
+
require 'active_support'
|
3
|
+
require 'active_support/core_ext/hash'
|
4
|
+
require "#{File.dirname(__FILE__)}/exception_handling_mailer"
|
5
|
+
|
6
|
+
EXCEPTION_HANDLING_MAILER_SEND_MAIL = true unless defined?(EXCEPTION_HANDLING_MAILER_SEND_MAIL)
|
7
|
+
|
8
|
+
_ = ActiveSupport::HashWithIndifferentAccess
|
9
|
+
|
10
|
+
if defined?(EVENTMACHINE_EXCEPTION_HANDLING) && EVENTMACHINE_EXCEPTION_HANDLING
|
11
|
+
require 'em/protocols/smtpclient'
|
12
|
+
end
|
13
|
+
|
14
|
+
module ExceptionHandling # never included
|
15
|
+
|
16
|
+
class Warning < StandardError; end
|
17
|
+
class MailerTimeout < Timeout::Error; end
|
18
|
+
class ClientLoggingError < StandardError; end
|
19
|
+
|
20
|
+
SUMMARY_THRESHOLD = 5
|
21
|
+
SUMMARY_PERIOD = 60*60 # 1.hour
|
22
|
+
|
23
|
+
|
24
|
+
SECTIONS = [:request, :session, :environment, :backtrace, :event_response]
|
25
|
+
EXCEPTION_FILTER_LIST_PATH = "#{defined?(Rails) ? Rails.root : '.'}/config/exception_filters.yml"
|
26
|
+
|
27
|
+
ENVIRONMENT_WHITELIST = [
|
28
|
+
/^HTTP_/,
|
29
|
+
/^QUERY_/,
|
30
|
+
/^REQUEST_/,
|
31
|
+
/^SERVER_/
|
32
|
+
]
|
33
|
+
|
34
|
+
ENVIRONMENT_OMIT =(
|
35
|
+
<<EOF
|
36
|
+
CONTENT_TYPE: application/x-www-form-urlencoded
|
37
|
+
GATEWAY_INTERFACE: CGI/1.2
|
38
|
+
HTTP_ACCEPT: */*
|
39
|
+
HTTP_ACCEPT: */*, text/javascript, text/html, application/xml, text/xml, */*
|
40
|
+
HTTP_ACCEPT_CHARSET: ISO-8859-1,utf-8;q=0.7,*;q=0.7
|
41
|
+
HTTP_ACCEPT_ENCODING: gzip, deflate
|
42
|
+
HTTP_ACCEPT_ENCODING: gzip,deflate
|
43
|
+
HTTP_ACCEPT_LANGUAGE: en-us
|
44
|
+
HTTP_CACHE_CONTROL: no-cache
|
45
|
+
HTTP_CONNECTION: Keep-Alive
|
46
|
+
HTTP_HOST: www.ringrevenue.com
|
47
|
+
HTTP_MAX_FORWARDS: 10
|
48
|
+
HTTP_UA_CPU: x86
|
49
|
+
HTTP_VERSION: HTTP/1.1
|
50
|
+
HTTP_X_FORWARDED_HOST: www.ringrevenue.com
|
51
|
+
HTTP_X_FORWARDED_SERVER: www2.ringrevenue.com
|
52
|
+
HTTP_X_REQUESTED_WITH: XMLHttpRequest
|
53
|
+
LANG:
|
54
|
+
LANG:
|
55
|
+
PATH: /sbin:/usr/sbin:/bin:/usr/bin
|
56
|
+
PWD: /
|
57
|
+
RAILS_ENV: production
|
58
|
+
RAW_POST_DATA: id=500
|
59
|
+
REMOTE_ADDR: 10.251.34.225
|
60
|
+
SCRIPT_NAME: /
|
61
|
+
SERVER_NAME: www.ringrevenue.com
|
62
|
+
SERVER_PORT: 80
|
63
|
+
SERVER_PROTOCOL: HTTP/1.1
|
64
|
+
SERVER_SOFTWARE: Mongrel 1.1.4
|
65
|
+
SHLVL: 1
|
66
|
+
TERM: linux
|
67
|
+
TERM: xterm-color
|
68
|
+
_: /usr/bin/mongrel_cluster_ctl
|
69
|
+
EOF
|
70
|
+
).split("\n")
|
71
|
+
|
72
|
+
AUTHENTICATION_HEADERS = ['HTTP_AUTHORIZATION','X-HTTP_AUTHORIZATION','X_HTTP_AUTHORIZATION','REDIRECT_X_HTTP_AUTHORIZATION']
|
73
|
+
|
74
|
+
@logger = Rails.logger if defined?(Rails)
|
75
|
+
|
76
|
+
|
77
|
+
class << self
|
78
|
+
attr_accessor :email_environment
|
79
|
+
attr_accessor :server_name
|
80
|
+
attr_accessor :sender_address
|
81
|
+
attr_accessor :exception_recipients
|
82
|
+
|
83
|
+
attr_accessor :current_controller
|
84
|
+
attr_accessor :last_exception_timestamp
|
85
|
+
attr_accessor :periodic_exception_intervals
|
86
|
+
attr_accessor :logger
|
87
|
+
|
88
|
+
#
|
89
|
+
# Gets called by Rack Middleware: DebugExceptions or ShowExceptions
|
90
|
+
# it does 2 things:
|
91
|
+
# log the error
|
92
|
+
# email the error
|
93
|
+
#
|
94
|
+
# but not during functional tests, when rack middleware is not used
|
95
|
+
#
|
96
|
+
def log_error_rack(exception, env, rack_filter)
|
97
|
+
timestamp = set_log_error_timestamp
|
98
|
+
exception_data = exception_to_data(exception, env, timestamp)
|
99
|
+
|
100
|
+
# TODO: add a more interesting custom description, like:
|
101
|
+
# custom_description = ": caught and processed by Rack middleware filter #{rack_filter}"
|
102
|
+
# which would be nice, but would also require changing quite a few tests
|
103
|
+
custom_description = ""
|
104
|
+
write_exception_to_log(exception, custom_description, timestamp)
|
105
|
+
|
106
|
+
if should_send_email?
|
107
|
+
controller = env['action_controller.instance']
|
108
|
+
# controller may not exist in some cases (like most 404 errors)
|
109
|
+
extract_and_merge_controller_data(controller, exception_data) if controller
|
110
|
+
log_error_email(exception_data, exception)
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
#
|
115
|
+
# Normal Operation:
|
116
|
+
# Called directly by our code, usually from rescue blocks.
|
117
|
+
# Does two things: write to log file and send an email
|
118
|
+
#
|
119
|
+
# Functional Test Operation:
|
120
|
+
# Calls into handle_stub_log_error and returns. no log file. no email.
|
121
|
+
#
|
122
|
+
def log_error(exception_or_string, exception_context = '', controller = nil, treat_as_local = false)
|
123
|
+
begin
|
124
|
+
ex = make_exception(exception_or_string)
|
125
|
+
timestamp = set_log_error_timestamp
|
126
|
+
data = exception_to_data(ex, exception_context, timestamp)
|
127
|
+
|
128
|
+
write_exception_to_log(ex, exception_context, timestamp)
|
129
|
+
|
130
|
+
if treat_as_local
|
131
|
+
return
|
132
|
+
end
|
133
|
+
|
134
|
+
if should_send_email?
|
135
|
+
controller ||= current_controller
|
136
|
+
|
137
|
+
if block_given?
|
138
|
+
# the expectation is that if the caller passed a block then they will be
|
139
|
+
# doing their own merge of hash values into data
|
140
|
+
begin
|
141
|
+
yield data
|
142
|
+
rescue Exception => ex
|
143
|
+
data.merge!(:environment => "Exception in yield: #{ex.class}:#{ex}")
|
144
|
+
end
|
145
|
+
elsif controller
|
146
|
+
# most of the time though, this method will not get passed a block
|
147
|
+
# and additional hash data is extracted from the controller
|
148
|
+
extract_and_merge_controller_data(controller, data)
|
149
|
+
end
|
150
|
+
|
151
|
+
log_error_email(data, ex)
|
152
|
+
end
|
153
|
+
|
154
|
+
rescue Exception => ex
|
155
|
+
$stderr.puts("ExceptionHandling.log_error rescued exception while logging #{exception_context}: #{exception_or_string}:\n#{ex.class}: #{ex}\n#{ex.backtrace.join("\n")}")
|
156
|
+
write_exception_to_log(ex, "ExceptionHandling.log_error rescued exception while logging #{exception_context}: #{exception_or_string}", timestamp)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
#
|
161
|
+
# Write an exception out to the log file using our own custom format.
|
162
|
+
#
|
163
|
+
def write_exception_to_log(ex, exception_context, timestamp)
|
164
|
+
ActiveSupport::Deprecation.silence do
|
165
|
+
ExceptionHandling.logger.fatal(
|
166
|
+
if ActionView::TemplateError === ex
|
167
|
+
"#{ex} Error:#{timestamp}"
|
168
|
+
else
|
169
|
+
"\n(Error:#{timestamp}) #{ex.class} #{exception_context} (#{ex.message}):\n " + clean_backtrace(ex).join("\n ") + "\n\n"
|
170
|
+
end
|
171
|
+
)
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
#
|
176
|
+
# Pull certain fields out of the controller and add to the data hash.
|
177
|
+
#
|
178
|
+
def extract_and_merge_controller_data(controller, data)
|
179
|
+
data[:request] = {
|
180
|
+
:params => controller.request.parameters.to_hash,
|
181
|
+
:rails_root => Rails.root,
|
182
|
+
:url => controller.complete_request_uri
|
183
|
+
}
|
184
|
+
data[:environment].merge!(controller.request.env.to_hash)
|
185
|
+
|
186
|
+
controller.session[:fault_in_session]
|
187
|
+
data[:session] = {
|
188
|
+
:key => controller.request.session_options[:id],
|
189
|
+
:data => controller.session.dup
|
190
|
+
}
|
191
|
+
end
|
192
|
+
|
193
|
+
def log_warning( message )
|
194
|
+
log_error( Warning.new(message) )
|
195
|
+
end
|
196
|
+
|
197
|
+
def log_info( message )
|
198
|
+
ExceptionHandling.logger.info( message )
|
199
|
+
end
|
200
|
+
|
201
|
+
def log_debug( message )
|
202
|
+
ExceptionHandling.logger.debug( message )
|
203
|
+
end
|
204
|
+
|
205
|
+
def ensure_safe( exception_context = "" )
|
206
|
+
yield
|
207
|
+
rescue => ex
|
208
|
+
log_error ex, exception_context
|
209
|
+
return nil
|
210
|
+
end
|
211
|
+
|
212
|
+
def ensure_escalation( email_subject )
|
213
|
+
begin
|
214
|
+
yield
|
215
|
+
rescue => ex
|
216
|
+
log_error ex
|
217
|
+
escalate(email_subject, ex, ExceptionHandling.last_exception_timestamp)
|
218
|
+
nil
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
def set_log_error_timestamp
|
223
|
+
ExceptionHandling.last_exception_timestamp = Time.now.to_i
|
224
|
+
end
|
225
|
+
|
226
|
+
def should_send_email?
|
227
|
+
defined?( EXCEPTION_HANDLING_MAILER_SEND_MAIL ) && EXCEPTION_HANDLING_MAILER_SEND_MAIL
|
228
|
+
end
|
229
|
+
|
230
|
+
def trace_timing(description)
|
231
|
+
result = nil
|
232
|
+
time = Benchmark.measure do
|
233
|
+
result = yield
|
234
|
+
end
|
235
|
+
log_info "#{description} %.4fs " % time.real
|
236
|
+
result
|
237
|
+
end
|
238
|
+
|
239
|
+
def log_periodically(exception_key, interval, message)
|
240
|
+
self.periodic_exception_intervals ||= {}
|
241
|
+
last_logged = self.periodic_exception_intervals[exception_key]
|
242
|
+
if !last_logged || ( (last_logged + interval) < Time.now )
|
243
|
+
log_error( message )
|
244
|
+
self.periodic_exception_intervals[exception_key] = Time.now
|
245
|
+
end
|
246
|
+
end
|
247
|
+
|
248
|
+
# TODO: fix test to not use this.
|
249
|
+
def enhance_exception_data(data)
|
250
|
+
|
251
|
+
end
|
252
|
+
|
253
|
+
private
|
254
|
+
|
255
|
+
def log_error_email( data, exc )
|
256
|
+
enhance_exception_data( data )
|
257
|
+
normalize_exception_data( data )
|
258
|
+
clean_exception_data( data )
|
259
|
+
|
260
|
+
SECTIONS.each { |section| add_to_s( data[section] ) if data[section].is_a? Hash }
|
261
|
+
|
262
|
+
if exception_filters.filtered?( data )
|
263
|
+
return
|
264
|
+
end
|
265
|
+
|
266
|
+
if summarize_exception( data ) == :Summarized
|
267
|
+
return
|
268
|
+
end
|
269
|
+
|
270
|
+
deliver(ExceptionHandling::Mailer.exception_notification(data))
|
271
|
+
|
272
|
+
if defined?(Errplane)
|
273
|
+
Errplane.transmit(exc, :custom_data => data) unless exc.is_a?(Warning)
|
274
|
+
end
|
275
|
+
|
276
|
+
nil
|
277
|
+
end
|
278
|
+
|
279
|
+
def escalate( email_subject, ex, timestamp )
|
280
|
+
data = exception_to_data( ex, nil, timestamp )
|
281
|
+
deliver(ExceptionHandling::Mailer.escalation_notification(email_subject, data))
|
282
|
+
end
|
283
|
+
|
284
|
+
def deliver(mail_object)
|
285
|
+
if defined?(EVENTMACHINE_EXCEPTION_HANDLING) && EVENTMACHINE_EXCEPTION_HANDLING
|
286
|
+
EventMachine.schedule do # in case we're running outside the reactor
|
287
|
+
async_send_method = EVENTMACHINE_EXCEPTION_HANDLING == :Synchrony ? :asend : :send
|
288
|
+
smtp_settings = ActionMailer::Base.smtp_settings
|
289
|
+
send_deferrable = EventMachine::Protocols::SmtpClient.__send__(
|
290
|
+
async_send_method,
|
291
|
+
{
|
292
|
+
:host => smtp_settings[:address],
|
293
|
+
:port => smtp_settings[:port],
|
294
|
+
:domain => smtp_settings[:domain],
|
295
|
+
:auth => {:type=>:plain, :username=>smtp_settings[:user_name], :password=>smtp_settings[:password]},
|
296
|
+
:from => mail_object['from'].to_s,
|
297
|
+
:to => mail_object['to'].to_s,
|
298
|
+
:body => mail_object.body.to_s
|
299
|
+
}
|
300
|
+
)
|
301
|
+
send_deferrable.errback { |err| ExceptionHandling.logger.fatal("Failed to email by SMTP: #{err.inspect}") }
|
302
|
+
end
|
303
|
+
else
|
304
|
+
safe_email_deliver do
|
305
|
+
mail_object.deliver
|
306
|
+
end
|
307
|
+
end
|
308
|
+
end
|
309
|
+
|
310
|
+
def safe_email_deliver
|
311
|
+
Timeout::timeout 30, MailerTimeout do
|
312
|
+
yield
|
313
|
+
end
|
314
|
+
rescue StandardError, MailerTimeout => ex
|
315
|
+
$stderr.puts("ExceptionHandling::safe_email_deliver rescued: #{ex.class}: #{ex}\n#{ex.backtrace.join("\n")}")
|
316
|
+
log_error( ex, "ExceptionHandling::safe_email_deliver", nil, true )
|
317
|
+
end
|
318
|
+
|
319
|
+
def clean_exception_data( data )
|
320
|
+
if (as_array = data[:backtrace].to_a).size == 1
|
321
|
+
data[:backtrace] = as_array.first.to_s.split(/\n\s*/)
|
322
|
+
end
|
323
|
+
|
324
|
+
if data[:request].is_a?(Hash) && data[:request][:params].is_a?(Hash)
|
325
|
+
data[:request][:params] = clean_params(data[:request][:params])
|
326
|
+
end
|
327
|
+
|
328
|
+
if data[:environment].is_a?(Hash)
|
329
|
+
data[:environment] = clean_environment(data[:environment])
|
330
|
+
end
|
331
|
+
end
|
332
|
+
|
333
|
+
def normalize_exception_data( data )
|
334
|
+
if data[:location].nil?
|
335
|
+
data[:location] = {}
|
336
|
+
if data[:request] && data[:request].key?( :params )
|
337
|
+
data[:location][:controller] = data[:request][:params]['controller']
|
338
|
+
data[:location][:action] = "fake action"
|
339
|
+
end
|
340
|
+
end
|
341
|
+
if data[:backtrace] && data[:backtrace].first
|
342
|
+
first_line = data[:backtrace].first
|
343
|
+
|
344
|
+
# template exceptions have the line number and filename as the first element in backtrace
|
345
|
+
if matched = first_line.match( /on line #(\d*) of (.*)/i )
|
346
|
+
backtrace_hash = {}
|
347
|
+
backtrace_hash[:line] = matched[1]
|
348
|
+
backtrace_hash[:file] = matched[2]
|
349
|
+
else
|
350
|
+
backtrace_hash = Hash[* [:file, :line].zip( first_line.split( ':' )[0..1]).flatten ]
|
351
|
+
end
|
352
|
+
|
353
|
+
data[:location].merge!( backtrace_hash )
|
354
|
+
end
|
355
|
+
end
|
356
|
+
|
357
|
+
def clean_params params
|
358
|
+
params.each do |k, v|
|
359
|
+
params[k] = "[FILTERED]" if k =~ /password/
|
360
|
+
end
|
361
|
+
end
|
362
|
+
|
363
|
+
def clean_environment env
|
364
|
+
Hash[ env.map do |k, v|
|
365
|
+
[k, v] if !"#{k}: #{v}".in?(ENVIRONMENT_OMIT) && ENVIRONMENT_WHITELIST.any? { |regex| k =~ regex }
|
366
|
+
end.compact ]
|
367
|
+
end
|
368
|
+
|
369
|
+
def exception_filters
|
370
|
+
@exception_filters ||= ExceptionFilters.new( EXCEPTION_FILTER_LIST_PATH )
|
371
|
+
end
|
372
|
+
|
373
|
+
def clean_backtrace(exception)
|
374
|
+
if exception.backtrace.nil?
|
375
|
+
['<no backtrace>']
|
376
|
+
elsif exception.is_a?(ClientLoggingError)
|
377
|
+
exception.backtrace
|
378
|
+
elsif defined?(Rails)
|
379
|
+
Rails.backtrace_cleaner.clean(exception.backtrace)
|
380
|
+
else
|
381
|
+
exception.backtrace
|
382
|
+
end
|
383
|
+
end
|
384
|
+
|
385
|
+
def clear_exception_summary
|
386
|
+
@last_exception = nil
|
387
|
+
end
|
388
|
+
|
389
|
+
# Returns :Summarized iff exception has been added to summary and therefore should not be sent.
|
390
|
+
def summarize_exception( data )
|
391
|
+
if @last_exception
|
392
|
+
same_signature = @last_exception[:backtrace] == data[:backtrace]
|
393
|
+
|
394
|
+
case @last_exception[:state]
|
395
|
+
|
396
|
+
when :NotSummarized
|
397
|
+
if same_signature
|
398
|
+
@last_exception[:count] += 1
|
399
|
+
if @last_exception[:count] >= SUMMARY_THRESHOLD
|
400
|
+
@last_exception.merge! :state => :Summarized, :first_seen => Time.now, :count => 0
|
401
|
+
end
|
402
|
+
return nil
|
403
|
+
end
|
404
|
+
|
405
|
+
when :Summarized
|
406
|
+
if same_signature
|
407
|
+
@last_exception[:count] += 1
|
408
|
+
if Time.now - @last_exception[:first_seen] > SUMMARY_PERIOD
|
409
|
+
send_exception_summary(data, @last_exception[:first_seen], @last_exception[:count])
|
410
|
+
@last_exception.merge! :first_seen => Time.now, :count => 0
|
411
|
+
end
|
412
|
+
return :Summarized
|
413
|
+
elsif @last_exception[:count] > 0 # send the left-over, if any
|
414
|
+
send_exception_summary(@last_exception[:data], @last_exception[:first_seen], @last_exception[:count])
|
415
|
+
end
|
416
|
+
|
417
|
+
else
|
418
|
+
raise "Unknown state #{@last_exception[:state]}"
|
419
|
+
end
|
420
|
+
end
|
421
|
+
|
422
|
+
# New signature we haven't seen before. Not summarized yet--we're just starting the count.
|
423
|
+
@last_exception = {
|
424
|
+
:data => data,
|
425
|
+
:count => 1,
|
426
|
+
:first_seen => Time.now,
|
427
|
+
:backtrace => data[:backtrace],
|
428
|
+
:state => :NotSummarized
|
429
|
+
}
|
430
|
+
nil
|
431
|
+
end
|
432
|
+
|
433
|
+
def send_exception_summary( exception_data, first_seen, occurrences )
|
434
|
+
Timeout::timeout 30, MailerTimeout do
|
435
|
+
deliver(ExceptionHandling::Mailer.exception_notification(exception_data, first_seen, occurrences))
|
436
|
+
end
|
437
|
+
rescue StandardError, MailerTimeout => ex
|
438
|
+
$stderr.puts("ExceptionHandling.log_error_email rescued exception while logging #{exception_context}: #{exception_or_string}:\n#{ex.class}: #{ex}\n#{ex.backtrace.join("\n")}")
|
439
|
+
log_error(ex, "ExceptionHandling::log_error_email rescued exception while logging #{exception_context}: #{exception_or_string}", nil, true)
|
440
|
+
end
|
441
|
+
|
442
|
+
def add_to_s( data_section )
|
443
|
+
data_section[:to_s] = dump_hash( data_section )
|
444
|
+
end
|
445
|
+
|
446
|
+
def exception_to_data( exception, exception_context, timestamp )
|
447
|
+
data = ActiveSupport::HashWithIndifferentAccess.new
|
448
|
+
data[:error_class] = exception.class.name
|
449
|
+
data[:error_string]= "#{data[:error_class]}: #{exception.message}"
|
450
|
+
data[:timestamp] = timestamp
|
451
|
+
data[:backtrace] = clean_backtrace(exception)
|
452
|
+
if exception_context && exception_context.is_a?(Hash)
|
453
|
+
# if we are a hash, then we got called from the DebugExceptions rack middleware filter
|
454
|
+
# and we need to do some things different to get the info we want
|
455
|
+
data[:error] = "#{data[:error_class]}: #{exception.message}"
|
456
|
+
data[:session] = exception_context['rack.session']
|
457
|
+
data[:environment] = exception_context
|
458
|
+
else
|
459
|
+
data[:error] = "#{data[:error_string]}#{': ' + exception_context unless exception_context.blank?}"
|
460
|
+
data[:environment] = { :message => exception_context }
|
461
|
+
end
|
462
|
+
data
|
463
|
+
end
|
464
|
+
|
465
|
+
def make_exception(exception_or_string)
|
466
|
+
if exception_or_string.is_a?(Exception)
|
467
|
+
exception_or_string
|
468
|
+
else
|
469
|
+
begin
|
470
|
+
# raise to capture a backtrace
|
471
|
+
raise StandardError, exception_or_string
|
472
|
+
rescue => ex
|
473
|
+
ex
|
474
|
+
end
|
475
|
+
end
|
476
|
+
end
|
477
|
+
|
478
|
+
def dump_hash( h, indent_level = 0 )
|
479
|
+
result = ""
|
480
|
+
h.sort { |a, b| a.to_s <=> b.to_s }.each do |key, value|
|
481
|
+
result << ' ' * (2 * indent_level)
|
482
|
+
result << "#{key}:"
|
483
|
+
case value
|
484
|
+
when Hash
|
485
|
+
result << "\n" << dump_hash( value, indent_level + 1 )
|
486
|
+
else
|
487
|
+
result << " #{value}\n"
|
488
|
+
end
|
489
|
+
end unless h.nil?
|
490
|
+
result
|
491
|
+
end
|
492
|
+
end
|
493
|
+
|
494
|
+
class ExceptionFilters
|
495
|
+
class Filter
|
496
|
+
def initialize filter_name, regexes
|
497
|
+
@regexes = Hash[ regexes.map do |section, regex|
|
498
|
+
section = section.to_sym
|
499
|
+
raise "Unknown section: #{section}" unless section == :error || section.in?( ExceptionHandling::SECTIONS )
|
500
|
+
[section, (Regexp.new(regex, 'i') unless regex.blank?)]
|
501
|
+
end ]
|
502
|
+
|
503
|
+
raise "Filter #{filter_name} has all blank regexes: #{regexes.inspect}" if @regexes.all? { |section, regex| regex.nil? }
|
504
|
+
end
|
505
|
+
|
506
|
+
def match?(exception_data)
|
507
|
+
@regexes.all? do |section, regex|
|
508
|
+
regex.nil? ||
|
509
|
+
case exception_data[section]
|
510
|
+
when String
|
511
|
+
regex =~ exception_data[section]
|
512
|
+
when Array
|
513
|
+
exception_data[section].any? { |row| row =~ regex }
|
514
|
+
when Hash
|
515
|
+
exception_data[section] && exception_data[section][:to_s] =~ regex
|
516
|
+
when NilClass
|
517
|
+
false
|
518
|
+
else
|
519
|
+
raise "Unexpected class #{exception_data[section].class.name}"
|
520
|
+
end
|
521
|
+
end
|
522
|
+
end
|
523
|
+
end
|
524
|
+
|
525
|
+
def initialize( filter_path )
|
526
|
+
@filter_path = filter_path
|
527
|
+
@filters = { }
|
528
|
+
@filters_last_modified_time = nil
|
529
|
+
end
|
530
|
+
|
531
|
+
def filtered?( exception_data )
|
532
|
+
refresh_filters
|
533
|
+
|
534
|
+
@filters.any? do |name, filter|
|
535
|
+
if ( match = filter.match?( exception_data ) )
|
536
|
+
ExceptionHandling.logger.warn( "Filtered exception using '#{name}'; not sending email to notify" )
|
537
|
+
end
|
538
|
+
match
|
539
|
+
end
|
540
|
+
end
|
541
|
+
|
542
|
+
private
|
543
|
+
|
544
|
+
def refresh_filters
|
545
|
+
mtime = last_modified_time
|
546
|
+
if @filters_last_modified_time.nil? || mtime != @filters_last_modified_time
|
547
|
+
ExceptionHandling.logger.info( "Reloading filter list from: #{@filter_path}. Last loaded time: #{@filters_last_modified_time}. Last modified time: #{mtime}" )
|
548
|
+
@filters_last_modified_time = mtime # make race condition fall on the side of reloading unnecessarily next time rather than missing a set of changes
|
549
|
+
|
550
|
+
@filters = load_file
|
551
|
+
end
|
552
|
+
|
553
|
+
rescue => ex # any exceptions
|
554
|
+
ExceptionHandling::log_error( ex, "ExceptionRegexes::refresh_filters: #{@filter_path}", nil, true)
|
555
|
+
end
|
556
|
+
|
557
|
+
def load_file
|
558
|
+
# store all strings from YAML file into regex's on initial load, instead of converting to regex on every exception that is logged
|
559
|
+
filters = YAML::load_file( @filter_path )
|
560
|
+
Hash[ filters.map do |filter_name, regexes|
|
561
|
+
[filter_name, Filter.new( filter_name, regexes )]
|
562
|
+
end ]
|
563
|
+
end
|
564
|
+
|
565
|
+
def last_modified_time
|
566
|
+
File.mtime( @filter_path )
|
567
|
+
end
|
568
|
+
end
|
569
|
+
|
570
|
+
|
571
|
+
public
|
572
|
+
|
573
|
+
module Methods # included on models and controllers
|
574
|
+
protected
|
575
|
+
def log_error(exception_or_string, exception_context = '')
|
576
|
+
controller = self if respond_to?(:request) && respond_to?(:session)
|
577
|
+
ExceptionHandling.log_error(exception_or_string, exception_context, controller)
|
578
|
+
end
|
579
|
+
|
580
|
+
def log_error_rack(exception_or_string, exception_context = '', rack_filter = '')
|
581
|
+
ExceptionHandling.log_error_rack(exception_or_string, exception_context, controller)
|
582
|
+
end
|
583
|
+
|
584
|
+
def log_warning(message)
|
585
|
+
log_error(Warning.new(message))
|
586
|
+
end
|
587
|
+
|
588
|
+
def log_info(message)
|
589
|
+
ExceptionHandling.logger.info( message )
|
590
|
+
end
|
591
|
+
|
592
|
+
def log_debug(message)
|
593
|
+
ExceptionHandling.logger.debug( message )
|
594
|
+
end
|
595
|
+
|
596
|
+
def ensure_safe(exception_context = "")
|
597
|
+
begin
|
598
|
+
yield
|
599
|
+
rescue => ex
|
600
|
+
log_error ex, exception_context
|
601
|
+
nil
|
602
|
+
end
|
603
|
+
end
|
604
|
+
|
605
|
+
def ensure_escalation(*args)
|
606
|
+
ExceptionHandling.ensure_escalation(*args) do
|
607
|
+
yield
|
608
|
+
end
|
609
|
+
end
|
610
|
+
|
611
|
+
# Store aside the current controller when included
|
612
|
+
LONG_REQUEST_SECONDS = (defined?(Rails) && Rails.env == 'test' ? 300 : 30)
|
613
|
+
def set_current_controller
|
614
|
+
ExceptionHandling.current_controller = self
|
615
|
+
result = nil
|
616
|
+
time = Benchmark.measure do
|
617
|
+
result = yield
|
618
|
+
end
|
619
|
+
name = " in #{controller_name}::#{action_name}" rescue " "
|
620
|
+
log_error( "Long controller action detected#{name} %.4fs " % time.real ) if time.real > LONG_REQUEST_SECONDS && !['development', 'test'].include?(Rails.env)
|
621
|
+
result
|
622
|
+
ensure
|
623
|
+
ExceptionHandling.current_controller = nil
|
624
|
+
end
|
625
|
+
|
626
|
+
def self.included( controller )
|
627
|
+
controller.around_filter :set_current_controller if controller.respond_to? :around_filter
|
628
|
+
end
|
629
|
+
end
|
630
|
+
end
|