right_support 1.4.1 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -24,10 +24,10 @@
24
24
  # A namespace to hold load-balancing policies to be used with RequestBalancer
25
25
  # and potentially other networking classes.
26
26
  #
27
- module RightSupport::Net::Balancing
27
+ module RightSupport::Net::LB
28
28
 
29
29
  end
30
30
 
31
- Dir[File.expand_path('../balancing/*.rb', __FILE__)].each do |filename|
32
- require filename
33
- end
31
+ require 'right_support/net/lb/health_check'
32
+ require 'right_support/net/lb/round_robin'
33
+ require 'right_support/net/lb/sticky'
@@ -22,7 +22,7 @@
22
22
 
23
23
  require 'set'
24
24
 
25
- module RightSupport::Net::Balancing
25
+ module RightSupport::Net::LB
26
26
 
27
27
  # TODO refactor this class. We store too much unstructured data about EPs; should have a simple
28
28
  # class representing EP state, and then perhaps move what logic remains into the HealthCheck class
@@ -20,7 +20,7 @@
20
20
  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
21
  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
22
 
23
- module RightSupport::Net::Balancing
23
+ module RightSupport::Net::LB
24
24
  class RoundRobin
25
25
 
26
26
  def initialize(options ={})
@@ -20,8 +20,7 @@
20
20
  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
21
  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
22
 
23
- module RightSupport::Net::Balancing
24
-
23
+ module RightSupport::Net::LB
25
24
  # Implementation concepts: Create a policy that selects an endpoint and sticks with it.
26
25
  #
27
26
  # The policy should:
@@ -30,7 +29,7 @@ module RightSupport::Net::Balancing
30
29
  # - re-iterate through each endpoint when it's endpoint loses validity;
31
30
  # - return an Exception if it performs a complete iteration though each endpoint and finds none valid;
32
31
 
33
- class StickyPolicy
32
+ class Sticky
34
33
 
35
34
  def initialize(options = {})
36
35
  @health_check = options.delete(:health_check)
@@ -108,7 +108,7 @@ module RightSupport::Net
108
108
  raise ArgumentError, "Must specify at least one endpoint"
109
109
  end
110
110
 
111
- @options[:policy] ||= RightSupport::Net::Balancing::RoundRobin
111
+ @options[:policy] ||= RightSupport::Net::LB::RoundRobin
112
112
  @policy = @options[:policy]
113
113
  @policy = @policy.new(options) if @policy.is_a?(Class)
114
114
 
@@ -29,5 +29,6 @@ module RightSupport
29
29
  end
30
30
  end
31
31
 
32
- require 'right_support/rack/custom_logger'
32
+ require 'right_support/rack/log_setter'
33
33
  require 'right_support/rack/request_logger'
34
+ require 'right_support/rack/request_tracker'
@@ -1,5 +1,5 @@
1
1
  #
2
- # Copyright (c) 2011 RightScale Inc
2
+ # Copyright (c) 2012 RightScale Inc
3
3
  #
4
4
  # Permission is hereby granted, free of charge, to any person obtaining
5
5
  # a copy of this software and associated documentation files (the
@@ -23,11 +23,15 @@
23
23
  require 'logger'
24
24
 
25
25
  module RightSupport::Rack
26
- # A Rack middleware that allows an arbitrary object to be used as the Rack logger.
27
- # This is more flexible than Rack's built-in Logger middleware, which always logs
28
- # to a file-based Logger and doesn't allow you to control anything other than the
29
- # filename.
30
- class CustomLogger
26
+ # A Rack middleware that logs information about every HTTP request received and
27
+ # every exception raised while processing a request.
28
+ #
29
+ # The middleware can be configured to use its own logger, but defaults to using
30
+ # env['rack.logger'] for logging if it is present. If 'rack.logger' is not set,
31
+ # this middleware will set it before calling the next middleware. Therefore,
32
+ # RequestLogger can be used standalone to fulfill all logging needs, or combined
33
+ # with Rack::Logger or another middleware that provides logging services.
34
+ class LogSetter
31
35
  # Initialize an instance of the middleware. For backward compatibility, the order of the
32
36
  # logger and level parameters can be switched.
33
37
  #
@@ -36,22 +40,9 @@ module RightSupport::Rack
36
40
  # logger(Logger):: (optional) the Logger object to use, defaults to a STDERR logger
37
41
  # level(Integer):: (optional) a Logger level-constant (INFO, ERROR) to set the logger to
38
42
  #
39
- def initialize(app, arg1=nil, arg2=nil)
40
- if arg1.is_a?(Integer)
41
- level = arg1
42
- elsif arg1.is_a?(Logger)
43
- logger = arg1
44
- end
45
-
46
- if arg2.is_a?(Integer)
47
- level = arg2
48
- elsif arg2.is_a?(Logger)
49
- logger = arg2
50
- end
51
-
43
+ def initialize(app, options={})
52
44
  @app = app
53
- @logger = logger
54
- @level = level
45
+ @logger = options[:logger]
55
46
  end
56
47
 
57
48
  # Add a logger to the Rack environment and call the next middleware.
@@ -62,18 +53,15 @@ module RightSupport::Rack
62
53
  # === Return
63
54
  # always returns whatever value is returned by the next layer of middleware
64
55
  def call(env)
65
- #emulate the behavior of Rack::CommonLogger middleware, which instantiates a
66
- #default logger if one has not been provided in the initializer
67
- @logger = ::Logger.new(env['rack.errors'] || STDERR) unless @logger
68
-
69
- if @level
70
- old_level = @logger.level
71
- @logger.level = @level
56
+ if @logger
57
+ logger = @logger
58
+ elsif env['rack.logger']
59
+ logger = env['rack.logger']
72
60
  end
73
- env['rack.logger'] = @logger
74
- status, header, body = @app.call(env)
75
- @logger.level = old_level if @level
76
- return [status, header, body]
61
+
62
+ env['rack.logger'] ||= logger
63
+
64
+ return @app.call(env)
77
65
  end
78
- end
66
+ end
79
67
  end
@@ -32,17 +32,13 @@ module RightSupport::Rack
32
32
  # RequestLogger can be used standalone to fulfill all logging needs, or combined
33
33
  # with Rack::Logger or another middleware that provides logging services.
34
34
  class RequestLogger
35
- # Initialize an instance of the middleware. For backward compatibility, the order of the
36
- # logger and level parameters can be switched.
35
+ # Initialize an instance of the middleware.
37
36
  #
38
37
  # === Parameters
39
38
  # app(Object):: the inner application or middleware layer; must respond to #call
40
- # logger(Logger):: (optional) the Logger object to use, defaults to a STDERR logger
41
- # level(Integer):: (optional) a Logger level-constant (INFO, ERROR) to set the logger to
42
39
  #
43
- def initialize(app, options={})
44
- @app = app
45
- @logger = options[:logger]
40
+ def initialize(app)
41
+ @app = app
46
42
  end
47
43
 
48
44
  # Add a logger to the Rack environment and call the next middleware.
@@ -53,17 +49,13 @@ module RightSupport::Rack
53
49
  # === Return
54
50
  # always returns whatever value is returned by the next layer of middleware
55
51
  def call(env)
56
- if @logger
57
- logger = @logger
58
- elsif env['rack.logger']
59
- logger = env['rack.logger']
60
- end
61
-
62
- env['rack.logger'] ||= logger
52
+ logger = env["rack.logger"]
63
53
 
64
54
  began_at = Time.now
55
+
56
+ log_request_begin(logger, env)
65
57
  status, header, body = @app.call(env)
66
- log_request(logger, env, status, began_at)
58
+ log_request_end(logger, env, status, began_at)
67
59
  log_exception(logger, env['sinatra.error']) if env['sinatra.error']
68
60
 
69
61
  return [status, header, body]
@@ -74,10 +66,15 @@ module RightSupport::Rack
74
66
 
75
67
  private
76
68
 
77
- # NON Logger functions below
78
- def log_request(logger, env, status, began_at)
79
- duration = Time.now - began_at
80
-
69
+ # Log beginning of request
70
+ #
71
+ # === Parameters
72
+ # logger(Object):: the Rack logger
73
+ # env(Hash):: the Rack environment
74
+ #
75
+ # === Return
76
+ # always returns true
77
+ def log_request_begin(logger, env)
81
78
  # Assuming remote addresses are IPv4, make them all align to the same width
82
79
  remote_addr = env['HTTP_X_FORWARDED_FOR'] || env["REMOTE_ADDR"] || "-"
83
80
  remote_addr = remote_addr.ljust(15)
@@ -96,13 +93,42 @@ module RightSupport::Rack
96
93
  env["PATH_INFO"],
97
94
  query_info,
98
95
  env["HTTP_VERSION"],
96
+ env["rack.request_uuid"] || ''
97
+ ]
98
+
99
+ logger.info %Q{Begin: %s "%s %s%s %s" %s} % params
100
+ end
101
+
102
+ # Log end of request
103
+ #
104
+ # === Parameters
105
+ # logger(Object):: the Rack logger
106
+ # env(Hash):: the Rack environment
107
+ # status(Fixnum):: status of the Rack request
108
+ # began_at(Time):: time of the Rack request begging
109
+ #
110
+ # === Return
111
+ # always returns true
112
+ def log_request_end(logger, env, status, began_at)
113
+ duration = Time.now - began_at
114
+
115
+ params = [
99
116
  status,
100
- duration
117
+ duration,
118
+ env["rack.request_uuid"] || ''
101
119
  ]
102
120
 
103
- logger.info %Q{%s "%s %s%s %s" %d %0.3f} % params
121
+ logger.info %Q{End: %d %0.3f %s} % params
104
122
  end
105
123
 
124
+ # Log exception
125
+ #
126
+ # === Parameters
127
+ # logger(Object):: the Rack logger
128
+ # e(Exception):: Exception to be logged
129
+ #
130
+ # === Return
131
+ # always returns true
106
132
  def log_exception(logger, e)
107
133
  msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n")
108
134
  logger.error(msg)
@@ -1,5 +1,5 @@
1
1
  #
2
- # Copyright (c) 2012 RightScale Inc
2
+ # Copyright (c) 2011 RightScale Inc
3
3
  #
4
4
  # Permission is hereby granted, free of charge, to any person obtaining
5
5
  # a copy of this software and associated documentation files (the
@@ -20,44 +20,44 @@
20
20
  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
21
  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
22
 
23
- module RightSupport::Config
24
-
25
- # Recursive(according to []) implementation of TrueClass.
26
- # To use, just do: true1 = RecTrueClass.new
27
- # now you can do true1['a'], true1['a']['b'], ...
28
- # and it will return
29
- class RecursiveTrueClass
30
-
31
- def initialize
32
- @value = true
33
- end
34
-
35
- def [](something)
36
- self
37
- end
38
-
39
- # supporting standart boolean
40
- # opeimplementation
41
- def ==(other)
42
- @value==other
43
- end
23
+ module RightSupport::Rack
24
+ # TODO docs
25
+ class RequestTracker
26
+ REQUEST_LINEAGE_UUID_HEADER = "HTTP_X_REQUEST_LINEAGE_UUID".freeze
27
+ REQUEST_UUID_HEADER = "X-Request-Uuid".freeze
28
+ REQUEST_UUID_ENV_NAME = "rack.request_uuid".freeze
29
+ UUID_SEPARATOR = " ".freeze
44
30
 
45
- def &(other)
46
- @value & other
31
+ # Make a new Request tracker.
32
+ #
33
+ # Tags the requset with a new request UUID
34
+ #
35
+ # === Parameters
36
+ # app(Rack client): application to run
37
+ def initialize(app)
38
+ @app = app
47
39
  end
48
40
 
49
- def ^(other)
50
- @value ^ other
51
- end
41
+ def call(env)
42
+ if env.has_key? REQUEST_LINEAGE_UUID_HEADER
43
+ request_uuid = env[REQUEST_LINEAGE_UUID_HEADER] + UUID_SEPARATOR +
44
+ generate_request_uuid
45
+ else
46
+ request_uuid = generate_request_uuid
47
+ end
52
48
 
53
- def to_s
54
- @value.to_s
55
- end
49
+ env[REQUEST_UUID_ENV_NAME] = request_uuid
50
+
51
+ status, headers, body = @app.call(env)
56
52
 
57
- def |(other)
58
- @value | other
53
+ headers[REQUEST_UUID_HEADER] = request_uuid
54
+ [status, headers,body]
59
55
  end
60
56
 
57
+
58
+ def generate_request_uuid
59
+ ::RightSupport::Data::UUID.generate
60
+ end
61
61
  end
62
62
 
63
- end
63
+ end
@@ -16,28 +16,6 @@ module RightSupport::Ruby
16
16
  rescue LoadError => e
17
17
  return false
18
18
  end
19
-
20
- # Attempt to require one or more source files; if the require succeeds (or
21
- # if the files have already been successfully required), yield to the block.
22
- #
23
- # This method is useful to conditionally define code depending on the availability
24
- # of gems or standard-library source files.
25
- #
26
- # === Parameters
27
- # Forwards all parameters transparently through to Kernel#require.
28
- #
29
- # === Block
30
- # The block will be called if the require succeeds (if it does not raise LoadError).
31
- #
32
- # === Return
33
- # Preserves the return value of Kernel#require (generally either true or false).
34
- def if_require_succeeds(*args)
35
- result = require(*args)
36
- yield if block_given?
37
- return result
38
- rescue LoadError => e
39
- return false
40
- end
41
19
  end
42
20
  end
43
21
 
@@ -33,6 +33,9 @@ module RightSupport
33
33
  # Maximum characters in sub-stat value line
34
34
  MAX_SUB_STAT_VALUE_WIDTH = 80
35
35
 
36
+ # Maximum characters displayed for exception message
37
+ MAX_EXCEPTION_MESSAGE_WIDTH = 60
38
+
36
39
  # Separator between stat name and stat value
37
40
  SEPARATOR = " : "
38
41
 
@@ -130,67 +133,37 @@ module RightSupport
130
133
  end
131
134
  end
132
135
 
133
- # Wrap string by breaking it into lines at the specified separators
134
- # Allow for presence of color encoding when measuring string length
136
+ # Wrap string by breaking it into lines at the specified separator
135
137
  #
136
138
  # === Parameters
137
139
  # string(String):: String to be wrapped
138
- # max_length(Integer|Array):: Maximum length of a line, or array containing
139
- # maximum length of first line followed by maximum for any subsequent line
140
- # including indent
141
- # indent(String):: Indentation for each line after first
142
- # separators(Regexp):: Separators where string can wrap
140
+ # max_length(Integer):: Maximum length of a line excluding indentation
141
+ # indent(String):: Indentation for each line
142
+ # separator(String):: Separator at which to make line breaks
143
143
  #
144
144
  # === Return
145
- # (String|Array):: Multi-line string
146
- def self.wrap(string, max_length, indent, separators)
147
- # Strip color encoding from string
148
- strip = lambda { |string| string.gsub(/\e\[[0-9]*m/, "") }
149
-
150
- # Split string at specified separators and return an array of arrays
151
- # containing string segments and associated separator
152
- split = lambda do |string, separators|
153
- head, sep, tail = string.partition(separators)
154
- if tail == ""
155
- [[head, sep]]
156
- else
157
- [[head, sep]].concat(split.call(tail, separators))
158
- end
159
- end
160
-
145
+ # (String):: Multi-line string
146
+ def self.wrap(string, max_length, indent, separator)
161
147
  all = []
162
- line = ["", last_sep = ""]
163
- limit = max_length.is_a?(Array) ? max_length[0] : max_length
164
- length = 0
165
- separators = /#{separators}/ if separators.is_a?(String)
166
- split.call(string, separators).each do |str, sep|
167
- if (length + strip.call(str).size + last_sep.size + sep.size) > limit
168
- all.push(line) unless line[0] == ""
169
- line = [indent, last_sep = ""]
170
- length = indent.size
171
- limit = max_length.is_a?(Array) ? max_length[1] : max_length
148
+ line = ""
149
+ for l in string.split(separator)
150
+ if (line + l).length >= max_length
151
+ all.push(line)
152
+ line = ""
172
153
  end
173
- line[0] += (str = last_sep + str)
174
- line[1] = last_sep = sep
175
- length += strip.call(str).size
154
+ line += line == "" ? l : separator + l
176
155
  end
177
- all.push(line)
178
- all[0..-2].inject("") { |a, (str, sep)| a + str + sep + "\n" } + all[-1][0]
156
+ all.push(line).join(separator + "\n" + indent)
179
157
  end
180
158
 
181
159
  # Format UTC time value
182
160
  #
183
161
  # === Parameters
184
162
  # time(Integer):: Time in seconds in Unix-epoch to be formatted
185
- # with_year(Boolean):: Whether to include year, defaults to false
186
163
  #
187
164
  # (String):: Formatted time string
188
- def self.time_at(time, with_year = false)
189
- if with_year
190
- Time.at(time).strftime("%a %b %d %H:%M:%S %Y")
191
- else
192
- Time.at(time).strftime("%a %b %d %H:%M:%S")
193
- end
165
+ def self.time_at(time)
166
+ Time.at(time).strftime("%a %b %d %H:%M:%S")
194
167
  end
195
168
 
196
169
  # Sort hash elements by key in ascending order into array of key/value pairs
@@ -219,9 +192,9 @@ module RightSupport
219
192
  # Converts server statistics to a displayable format
220
193
  #
221
194
  # === Parameters
222
- # stats(Hash):: Statistics with generic keys "name", "identity", "hostname", "revision", "service uptime",
195
+ # stats(Hash):: Statistics with generic keys "name", "identity", "hostname", "service uptime",
223
196
  # "machine uptime", "memory KB", "stat time", "last reset time", "version", and "broker" with
224
- # the latter two and "revision", "machine uptime", "memory KB", "version", and "broker" being optional;
197
+ # the latter two and "machine uptime", "memory KB", "version", and "broker" being optional;
225
198
  # any other keys ending with "stats" have an associated hash value that is displayed in sorted
226
199
  # key order, unless "stats" is preceded by a non-blank, in which case that character is prepended
227
200
  # to the key to drive the sort order
@@ -229,6 +202,7 @@ module RightSupport
229
202
  # :name_width(Integer):: Maximum characters in displayed stat name
230
203
  # :sub_name_width(Integer):: Maximum characters in displayed sub-stat name
231
204
  # :sub_stat_value_width(Integer):: Maximum characters in displayed sub-stat value line
205
+ # :exception_message_width(Integer):: Maximum characters displayed for exception message
232
206
  #
233
207
  # === Return
234
208
  # (String):: Display string
@@ -237,12 +211,9 @@ module RightSupport
237
211
 
238
212
  str = stats["name"] ? sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "name", stats["name"]) : ""
239
213
  str += sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "identity", stats["identity"]) +
240
- sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "hostname", stats["hostname"])
241
- if stats.has_key?("revision")
242
- str += sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "revision", stats["revision"])
243
- end
244
- str += sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "stat time", time_at(stats["stat time"], with_year = true)) +
245
- sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "last reset", time_at(stats["last reset time"], with_year = true)) +
214
+ sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "hostname", stats["hostname"]) +
215
+ sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "stat time", time_at(stats["stat time"])) +
216
+ sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "last reset", time_at(stats["last reset time"])) +
246
217
  sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "service up", elapsed(stats["service uptime"]))
247
218
  if stats.has_key?("machine uptime")
248
219
  str += sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "machine up", elapsed(stats["machine uptime"]))
@@ -285,6 +256,7 @@ module RightSupport
285
256
  # :name_width(Integer):: Fixed width for left-justified name display
286
257
  # :sub_name_width(Integer):: Maximum characters in displayed sub-stat name
287
258
  # :sub_stat_value_width(Integer):: Maximum characters in displayed sub-stat value line
259
+ # :exception_message_width(Integer):: Maximum characters displayed for exception message
288
260
  #
289
261
  # === Return
290
262
  # str(String):: Broker display with one line per broker plus exceptions
@@ -330,8 +302,7 @@ module RightSupport
330
302
  str += if brokers["returns"].nil? || brokers["returns"].empty?
331
303
  "none\n"
332
304
  else
333
- wrap(activity_str(brokers["returns"]), [sub_stat_value_width, sub_stat_value_width + sub_value_indent.size],
334
- sub_value_indent, /, /) + "\n"
305
+ wrap(activity_str(brokers["returns"]), sub_stat_value_width, sub_value_indent, ", ") + "\n"
335
306
  end
336
307
  end
337
308
 
@@ -354,6 +325,7 @@ module RightSupport
354
325
  # :name_width(Integer):: Fixed width for left-justified name display
355
326
  # :sub_name_width(Integer):: Maximum characters in displayed sub-stat name
356
327
  # :sub_stat_value_width(Integer):: Maximum characters in displayed sub-stat value line
328
+ # :exception_message_width(Integer):: Maximum characters displayed for exception message
357
329
  #
358
330
  # === Return
359
331
  # (String):: Single line display of stat
@@ -377,15 +349,13 @@ module RightSupport
377
349
  if v.empty? || v["total"] == 0
378
350
  "none"
379
351
  elsif v["total"]
380
- wrap(activity_str(v), [sub_stat_value_width, sub_stat_value_width + sub_value_indent.size],
381
- sub_value_indent, ", ")
352
+ wrap(activity_str(v), sub_stat_value_width, sub_value_indent, ", ")
382
353
  elsif k =~ /last$/
383
354
  last_activity_str(v)
384
355
  elsif k == "exceptions"
385
356
  exceptions_str(v, sub_value_indent, options)
386
357
  else
387
- wrap(hash_str(v), [sub_stat_value_width, sub_stat_value_width + sub_value_indent.size],
388
- sub_value_indent, /, /)
358
+ wrap(hash_str(v), sub_stat_value_width, sub_value_indent, ", ")
389
359
  end
390
360
  else
391
361
  "#{v || "none"}"
@@ -457,19 +427,20 @@ module RightSupport
457
427
  # "recent"(Array):: Most recent as a hash of "count", "type", "message", "when", and "where"
458
428
  # indent(String):: Indentation for each line
459
429
  # options(Hash):: Formatting options
460
- # :sub_stat_value_width(Integer):: Maximum characters in displayed sub-stat value line
430
+ # :exception_message_width(Integer):: Maximum characters displayed for exception message
461
431
  #
462
432
  # === Return
463
433
  # (String):: Exceptions in displayable format with line separators
464
434
  def self.exceptions_str(exceptions, indent, options = {})
465
- sub_stat_value_width = options[:sub_stat_value_width] || MAX_SUB_STAT_VALUE_WIDTH
435
+ exception_message_width = options[:exception_message_width] || MAX_EXCEPTION_MESSAGE_WIDTH
466
436
  indent2 = indent + (" " * 4)
467
437
  exceptions.to_a.sort.map do |k, v|
468
438
  sprintf("%s total: %d, most recent:\n", k, v["total"]) + v["recent"].reverse.map do |e|
469
- where = " IN #{e["where"]}" if e["where"]
470
- indent + wrap("(#{e["count"]}) #{time_at(e["when"])} #{e["type"]}: #{e["message"]}#{where}",
471
- [sub_stat_value_width, sub_stat_value_width + indent2.size],
472
- indent2, / |\/\/|\/|::|\.|-/)
439
+ message = e["message"]
440
+ if message && message.size > (exception_message_width - 3)
441
+ message = e["message"][0, exception_message_width - 3] + "..."
442
+ end
443
+ indent + "(#{e["count"]}) #{time_at(e["when"])} #{e["type"]}: #{message}\n" + indent2 + "#{e["where"]}"
473
444
  end.join("\n")
474
445
  end.join("\n" + indent)
475
446
  end