right_support 1.4.1 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -24,10 +24,10 @@
24
24
  # A namespace to hold load-balancing policies to be used with RequestBalancer
25
25
  # and potentially other networking classes.
26
26
  #
27
- module RightSupport::Net::Balancing
27
+ module RightSupport::Net::LB
28
28
 
29
29
  end
30
30
 
31
- Dir[File.expand_path('../balancing/*.rb', __FILE__)].each do |filename|
32
- require filename
33
- end
31
+ require 'right_support/net/lb/health_check'
32
+ require 'right_support/net/lb/round_robin'
33
+ require 'right_support/net/lb/sticky'
@@ -22,7 +22,7 @@
22
22
 
23
23
  require 'set'
24
24
 
25
- module RightSupport::Net::Balancing
25
+ module RightSupport::Net::LB
26
26
 
27
27
  # TODO refactor this class. We store too much unstructured data about EPs; should have a simple
28
28
  # class representing EP state, and then perhaps move what logic remains into the HealthCheck class
@@ -20,7 +20,7 @@
20
20
  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
21
  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
22
 
23
- module RightSupport::Net::Balancing
23
+ module RightSupport::Net::LB
24
24
  class RoundRobin
25
25
 
26
26
  def initialize(options ={})
@@ -20,8 +20,7 @@
20
20
  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
21
  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
22
 
23
- module RightSupport::Net::Balancing
24
-
23
+ module RightSupport::Net::LB
25
24
  # Implementation concepts: Create a policy that selects an endpoint and sticks with it.
26
25
  #
27
26
  # The policy should:
@@ -30,7 +29,7 @@ module RightSupport::Net::Balancing
30
29
  # - re-iterate through each endpoint when it's endpoint loses validity;
31
30
  # - return an Exception if it performs a complete iteration though each endpoint and finds none valid;
32
31
 
33
- class StickyPolicy
32
+ class Sticky
34
33
 
35
34
  def initialize(options = {})
36
35
  @health_check = options.delete(:health_check)
@@ -108,7 +108,7 @@ module RightSupport::Net
108
108
  raise ArgumentError, "Must specify at least one endpoint"
109
109
  end
110
110
 
111
- @options[:policy] ||= RightSupport::Net::Balancing::RoundRobin
111
+ @options[:policy] ||= RightSupport::Net::LB::RoundRobin
112
112
  @policy = @options[:policy]
113
113
  @policy = @policy.new(options) if @policy.is_a?(Class)
114
114
 
@@ -29,5 +29,6 @@ module RightSupport
29
29
  end
30
30
  end
31
31
 
32
- require 'right_support/rack/custom_logger'
32
+ require 'right_support/rack/log_setter'
33
33
  require 'right_support/rack/request_logger'
34
+ require 'right_support/rack/request_tracker'
@@ -1,5 +1,5 @@
1
1
  #
2
- # Copyright (c) 2011 RightScale Inc
2
+ # Copyright (c) 2012 RightScale Inc
3
3
  #
4
4
  # Permission is hereby granted, free of charge, to any person obtaining
5
5
  # a copy of this software and associated documentation files (the
@@ -23,11 +23,15 @@
23
23
  require 'logger'
24
24
 
25
25
  module RightSupport::Rack
26
- # A Rack middleware that allows an arbitrary object to be used as the Rack logger.
27
- # This is more flexible than Rack's built-in Logger middleware, which always logs
28
- # to a file-based Logger and doesn't allow you to control anything other than the
29
- # filename.
30
- class CustomLogger
26
+ # A Rack middleware that logs information about every HTTP request received and
27
+ # every exception raised while processing a request.
28
+ #
29
+ # The middleware can be configured to use its own logger, but defaults to using
30
+ # env['rack.logger'] for logging if it is present. If 'rack.logger' is not set,
31
+ # this middleware will set it before calling the next middleware. Therefore,
32
+ # RequestLogger can be used standalone to fulfill all logging needs, or combined
33
+ # with Rack::Logger or another middleware that provides logging services.
34
+ class LogSetter
31
35
  # Initialize an instance of the middleware. For backward compatibility, the order of the
32
36
  # logger and level parameters can be switched.
33
37
  #
@@ -36,22 +40,9 @@ module RightSupport::Rack
36
40
  # logger(Logger):: (optional) the Logger object to use, defaults to a STDERR logger
37
41
  # level(Integer):: (optional) a Logger level-constant (INFO, ERROR) to set the logger to
38
42
  #
39
- def initialize(app, arg1=nil, arg2=nil)
40
- if arg1.is_a?(Integer)
41
- level = arg1
42
- elsif arg1.is_a?(Logger)
43
- logger = arg1
44
- end
45
-
46
- if arg2.is_a?(Integer)
47
- level = arg2
48
- elsif arg2.is_a?(Logger)
49
- logger = arg2
50
- end
51
-
43
+ def initialize(app, options={})
52
44
  @app = app
53
- @logger = logger
54
- @level = level
45
+ @logger = options[:logger]
55
46
  end
56
47
 
57
48
  # Add a logger to the Rack environment and call the next middleware.
@@ -62,18 +53,15 @@ module RightSupport::Rack
62
53
  # === Return
63
54
  # always returns whatever value is returned by the next layer of middleware
64
55
  def call(env)
65
- #emulate the behavior of Rack::CommonLogger middleware, which instantiates a
66
- #default logger if one has not been provided in the initializer
67
- @logger = ::Logger.new(env['rack.errors'] || STDERR) unless @logger
68
-
69
- if @level
70
- old_level = @logger.level
71
- @logger.level = @level
56
+ if @logger
57
+ logger = @logger
58
+ elsif env['rack.logger']
59
+ logger = env['rack.logger']
72
60
  end
73
- env['rack.logger'] = @logger
74
- status, header, body = @app.call(env)
75
- @logger.level = old_level if @level
76
- return [status, header, body]
61
+
62
+ env['rack.logger'] ||= logger
63
+
64
+ return @app.call(env)
77
65
  end
78
- end
66
+ end
79
67
  end
@@ -32,17 +32,13 @@ module RightSupport::Rack
32
32
  # RequestLogger can be used standalone to fulfill all logging needs, or combined
33
33
  # with Rack::Logger or another middleware that provides logging services.
34
34
  class RequestLogger
35
- # Initialize an instance of the middleware. For backward compatibility, the order of the
36
- # logger and level parameters can be switched.
35
+ # Initialize an instance of the middleware.
37
36
  #
38
37
  # === Parameters
39
38
  # app(Object):: the inner application or middleware layer; must respond to #call
40
- # logger(Logger):: (optional) the Logger object to use, defaults to a STDERR logger
41
- # level(Integer):: (optional) a Logger level-constant (INFO, ERROR) to set the logger to
42
39
  #
43
- def initialize(app, options={})
44
- @app = app
45
- @logger = options[:logger]
40
+ def initialize(app)
41
+ @app = app
46
42
  end
47
43
 
48
44
  # Add a logger to the Rack environment and call the next middleware.
@@ -53,17 +49,13 @@ module RightSupport::Rack
53
49
  # === Return
54
50
  # always returns whatever value is returned by the next layer of middleware
55
51
  def call(env)
56
- if @logger
57
- logger = @logger
58
- elsif env['rack.logger']
59
- logger = env['rack.logger']
60
- end
61
-
62
- env['rack.logger'] ||= logger
52
+ logger = env["rack.logger"]
63
53
 
64
54
  began_at = Time.now
55
+
56
+ log_request_begin(logger, env)
65
57
  status, header, body = @app.call(env)
66
- log_request(logger, env, status, began_at)
58
+ log_request_end(logger, env, status, began_at)
67
59
  log_exception(logger, env['sinatra.error']) if env['sinatra.error']
68
60
 
69
61
  return [status, header, body]
@@ -74,10 +66,15 @@ module RightSupport::Rack
74
66
 
75
67
  private
76
68
 
77
- # NON Logger functions below
78
- def log_request(logger, env, status, began_at)
79
- duration = Time.now - began_at
80
-
69
+ # Log beginning of request
70
+ #
71
+ # === Parameters
72
+ # logger(Object):: the Rack logger
73
+ # env(Hash):: the Rack environment
74
+ #
75
+ # === Return
76
+ # always returns true
77
+ def log_request_begin(logger, env)
81
78
  # Assuming remote addresses are IPv4, make them all align to the same width
82
79
  remote_addr = env['HTTP_X_FORWARDED_FOR'] || env["REMOTE_ADDR"] || "-"
83
80
  remote_addr = remote_addr.ljust(15)
@@ -96,13 +93,42 @@ module RightSupport::Rack
96
93
  env["PATH_INFO"],
97
94
  query_info,
98
95
  env["HTTP_VERSION"],
96
+ env["rack.request_uuid"] || ''
97
+ ]
98
+
99
+ logger.info %Q{Begin: %s "%s %s%s %s" %s} % params
100
+ end
101
+
102
+ # Log end of request
103
+ #
104
+ # === Parameters
105
+ # logger(Object):: the Rack logger
106
+ # env(Hash):: the Rack environment
107
+ # status(Fixnum):: status of the Rack request
108
+ # began_at(Time):: time of the Rack request begging
109
+ #
110
+ # === Return
111
+ # always returns true
112
+ def log_request_end(logger, env, status, began_at)
113
+ duration = Time.now - began_at
114
+
115
+ params = [
99
116
  status,
100
- duration
117
+ duration,
118
+ env["rack.request_uuid"] || ''
101
119
  ]
102
120
 
103
- logger.info %Q{%s "%s %s%s %s" %d %0.3f} % params
121
+ logger.info %Q{End: %d %0.3f %s} % params
104
122
  end
105
123
 
124
+ # Log exception
125
+ #
126
+ # === Parameters
127
+ # logger(Object):: the Rack logger
128
+ # e(Exception):: Exception to be logged
129
+ #
130
+ # === Return
131
+ # always returns true
106
132
  def log_exception(logger, e)
107
133
  msg = ["#{e.class} - #{e.message}", *e.backtrace].join("\n")
108
134
  logger.error(msg)
@@ -1,5 +1,5 @@
1
1
  #
2
- # Copyright (c) 2012 RightScale Inc
2
+ # Copyright (c) 2011 RightScale Inc
3
3
  #
4
4
  # Permission is hereby granted, free of charge, to any person obtaining
5
5
  # a copy of this software and associated documentation files (the
@@ -20,44 +20,44 @@
20
20
  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
21
  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
22
 
23
- module RightSupport::Config
24
-
25
- # Recursive(according to []) implementation of TrueClass.
26
- # To use, just do: true1 = RecTrueClass.new
27
- # now you can do true1['a'], true1['a']['b'], ...
28
- # and it will return
29
- class RecursiveTrueClass
30
-
31
- def initialize
32
- @value = true
33
- end
34
-
35
- def [](something)
36
- self
37
- end
38
-
39
- # supporting standart boolean
40
- # opeimplementation
41
- def ==(other)
42
- @value==other
43
- end
23
+ module RightSupport::Rack
24
+ # TODO docs
25
+ class RequestTracker
26
+ REQUEST_LINEAGE_UUID_HEADER = "HTTP_X_REQUEST_LINEAGE_UUID".freeze
27
+ REQUEST_UUID_HEADER = "X-Request-Uuid".freeze
28
+ REQUEST_UUID_ENV_NAME = "rack.request_uuid".freeze
29
+ UUID_SEPARATOR = " ".freeze
44
30
 
45
- def &(other)
46
- @value & other
31
+ # Make a new Request tracker.
32
+ #
33
+ # Tags the requset with a new request UUID
34
+ #
35
+ # === Parameters
36
+ # app(Rack client): application to run
37
+ def initialize(app)
38
+ @app = app
47
39
  end
48
40
 
49
- def ^(other)
50
- @value ^ other
51
- end
41
+ def call(env)
42
+ if env.has_key? REQUEST_LINEAGE_UUID_HEADER
43
+ request_uuid = env[REQUEST_LINEAGE_UUID_HEADER] + UUID_SEPARATOR +
44
+ generate_request_uuid
45
+ else
46
+ request_uuid = generate_request_uuid
47
+ end
52
48
 
53
- def to_s
54
- @value.to_s
55
- end
49
+ env[REQUEST_UUID_ENV_NAME] = request_uuid
50
+
51
+ status, headers, body = @app.call(env)
56
52
 
57
- def |(other)
58
- @value | other
53
+ headers[REQUEST_UUID_HEADER] = request_uuid
54
+ [status, headers,body]
59
55
  end
60
56
 
57
+
58
+ def generate_request_uuid
59
+ ::RightSupport::Data::UUID.generate
60
+ end
61
61
  end
62
62
 
63
- end
63
+ end
@@ -16,28 +16,6 @@ module RightSupport::Ruby
16
16
  rescue LoadError => e
17
17
  return false
18
18
  end
19
-
20
- # Attempt to require one or more source files; if the require succeeds (or
21
- # if the files have already been successfully required), yield to the block.
22
- #
23
- # This method is useful to conditionally define code depending on the availability
24
- # of gems or standard-library source files.
25
- #
26
- # === Parameters
27
- # Forwards all parameters transparently through to Kernel#require.
28
- #
29
- # === Block
30
- # The block will be called if the require succeeds (if it does not raise LoadError).
31
- #
32
- # === Return
33
- # Preserves the return value of Kernel#require (generally either true or false).
34
- def if_require_succeeds(*args)
35
- result = require(*args)
36
- yield if block_given?
37
- return result
38
- rescue LoadError => e
39
- return false
40
- end
41
19
  end
42
20
  end
43
21
 
@@ -33,6 +33,9 @@ module RightSupport
33
33
  # Maximum characters in sub-stat value line
34
34
  MAX_SUB_STAT_VALUE_WIDTH = 80
35
35
 
36
+ # Maximum characters displayed for exception message
37
+ MAX_EXCEPTION_MESSAGE_WIDTH = 60
38
+
36
39
  # Separator between stat name and stat value
37
40
  SEPARATOR = " : "
38
41
 
@@ -130,67 +133,37 @@ module RightSupport
130
133
  end
131
134
  end
132
135
 
133
- # Wrap string by breaking it into lines at the specified separators
134
- # Allow for presence of color encoding when measuring string length
136
+ # Wrap string by breaking it into lines at the specified separator
135
137
  #
136
138
  # === Parameters
137
139
  # string(String):: String to be wrapped
138
- # max_length(Integer|Array):: Maximum length of a line, or array containing
139
- # maximum length of first line followed by maximum for any subsequent line
140
- # including indent
141
- # indent(String):: Indentation for each line after first
142
- # separators(Regexp):: Separators where string can wrap
140
+ # max_length(Integer):: Maximum length of a line excluding indentation
141
+ # indent(String):: Indentation for each line
142
+ # separator(String):: Separator at which to make line breaks
143
143
  #
144
144
  # === Return
145
- # (String|Array):: Multi-line string
146
- def self.wrap(string, max_length, indent, separators)
147
- # Strip color encoding from string
148
- strip = lambda { |string| string.gsub(/\e\[[0-9]*m/, "") }
149
-
150
- # Split string at specified separators and return an array of arrays
151
- # containing string segments and associated separator
152
- split = lambda do |string, separators|
153
- head, sep, tail = string.partition(separators)
154
- if tail == ""
155
- [[head, sep]]
156
- else
157
- [[head, sep]].concat(split.call(tail, separators))
158
- end
159
- end
160
-
145
+ # (String):: Multi-line string
146
+ def self.wrap(string, max_length, indent, separator)
161
147
  all = []
162
- line = ["", last_sep = ""]
163
- limit = max_length.is_a?(Array) ? max_length[0] : max_length
164
- length = 0
165
- separators = /#{separators}/ if separators.is_a?(String)
166
- split.call(string, separators).each do |str, sep|
167
- if (length + strip.call(str).size + last_sep.size + sep.size) > limit
168
- all.push(line) unless line[0] == ""
169
- line = [indent, last_sep = ""]
170
- length = indent.size
171
- limit = max_length.is_a?(Array) ? max_length[1] : max_length
148
+ line = ""
149
+ for l in string.split(separator)
150
+ if (line + l).length >= max_length
151
+ all.push(line)
152
+ line = ""
172
153
  end
173
- line[0] += (str = last_sep + str)
174
- line[1] = last_sep = sep
175
- length += strip.call(str).size
154
+ line += line == "" ? l : separator + l
176
155
  end
177
- all.push(line)
178
- all[0..-2].inject("") { |a, (str, sep)| a + str + sep + "\n" } + all[-1][0]
156
+ all.push(line).join(separator + "\n" + indent)
179
157
  end
180
158
 
181
159
  # Format UTC time value
182
160
  #
183
161
  # === Parameters
184
162
  # time(Integer):: Time in seconds in Unix-epoch to be formatted
185
- # with_year(Boolean):: Whether to include year, defaults to false
186
163
  #
187
164
  # (String):: Formatted time string
188
- def self.time_at(time, with_year = false)
189
- if with_year
190
- Time.at(time).strftime("%a %b %d %H:%M:%S %Y")
191
- else
192
- Time.at(time).strftime("%a %b %d %H:%M:%S")
193
- end
165
+ def self.time_at(time)
166
+ Time.at(time).strftime("%a %b %d %H:%M:%S")
194
167
  end
195
168
 
196
169
  # Sort hash elements by key in ascending order into array of key/value pairs
@@ -219,9 +192,9 @@ module RightSupport
219
192
  # Converts server statistics to a displayable format
220
193
  #
221
194
  # === Parameters
222
- # stats(Hash):: Statistics with generic keys "name", "identity", "hostname", "revision", "service uptime",
195
+ # stats(Hash):: Statistics with generic keys "name", "identity", "hostname", "service uptime",
223
196
  # "machine uptime", "memory KB", "stat time", "last reset time", "version", and "broker" with
224
- # the latter two and "revision", "machine uptime", "memory KB", "version", and "broker" being optional;
197
+ # the latter two and "machine uptime", "memory KB", "version", and "broker" being optional;
225
198
  # any other keys ending with "stats" have an associated hash value that is displayed in sorted
226
199
  # key order, unless "stats" is preceded by a non-blank, in which case that character is prepended
227
200
  # to the key to drive the sort order
@@ -229,6 +202,7 @@ module RightSupport
229
202
  # :name_width(Integer):: Maximum characters in displayed stat name
230
203
  # :sub_name_width(Integer):: Maximum characters in displayed sub-stat name
231
204
  # :sub_stat_value_width(Integer):: Maximum characters in displayed sub-stat value line
205
+ # :exception_message_width(Integer):: Maximum characters displayed for exception message
232
206
  #
233
207
  # === Return
234
208
  # (String):: Display string
@@ -237,12 +211,9 @@ module RightSupport
237
211
 
238
212
  str = stats["name"] ? sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "name", stats["name"]) : ""
239
213
  str += sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "identity", stats["identity"]) +
240
- sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "hostname", stats["hostname"])
241
- if stats.has_key?("revision")
242
- str += sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "revision", stats["revision"])
243
- end
244
- str += sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "stat time", time_at(stats["stat time"], with_year = true)) +
245
- sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "last reset", time_at(stats["last reset time"], with_year = true)) +
214
+ sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "hostname", stats["hostname"]) +
215
+ sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "stat time", time_at(stats["stat time"])) +
216
+ sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "last reset", time_at(stats["last reset time"])) +
246
217
  sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "service up", elapsed(stats["service uptime"]))
247
218
  if stats.has_key?("machine uptime")
248
219
  str += sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "machine up", elapsed(stats["machine uptime"]))
@@ -285,6 +256,7 @@ module RightSupport
285
256
  # :name_width(Integer):: Fixed width for left-justified name display
286
257
  # :sub_name_width(Integer):: Maximum characters in displayed sub-stat name
287
258
  # :sub_stat_value_width(Integer):: Maximum characters in displayed sub-stat value line
259
+ # :exception_message_width(Integer):: Maximum characters displayed for exception message
288
260
  #
289
261
  # === Return
290
262
  # str(String):: Broker display with one line per broker plus exceptions
@@ -330,8 +302,7 @@ module RightSupport
330
302
  str += if brokers["returns"].nil? || brokers["returns"].empty?
331
303
  "none\n"
332
304
  else
333
- wrap(activity_str(brokers["returns"]), [sub_stat_value_width, sub_stat_value_width + sub_value_indent.size],
334
- sub_value_indent, /, /) + "\n"
305
+ wrap(activity_str(brokers["returns"]), sub_stat_value_width, sub_value_indent, ", ") + "\n"
335
306
  end
336
307
  end
337
308
 
@@ -354,6 +325,7 @@ module RightSupport
354
325
  # :name_width(Integer):: Fixed width for left-justified name display
355
326
  # :sub_name_width(Integer):: Maximum characters in displayed sub-stat name
356
327
  # :sub_stat_value_width(Integer):: Maximum characters in displayed sub-stat value line
328
+ # :exception_message_width(Integer):: Maximum characters displayed for exception message
357
329
  #
358
330
  # === Return
359
331
  # (String):: Single line display of stat
@@ -377,15 +349,13 @@ module RightSupport
377
349
  if v.empty? || v["total"] == 0
378
350
  "none"
379
351
  elsif v["total"]
380
- wrap(activity_str(v), [sub_stat_value_width, sub_stat_value_width + sub_value_indent.size],
381
- sub_value_indent, ", ")
352
+ wrap(activity_str(v), sub_stat_value_width, sub_value_indent, ", ")
382
353
  elsif k =~ /last$/
383
354
  last_activity_str(v)
384
355
  elsif k == "exceptions"
385
356
  exceptions_str(v, sub_value_indent, options)
386
357
  else
387
- wrap(hash_str(v), [sub_stat_value_width, sub_stat_value_width + sub_value_indent.size],
388
- sub_value_indent, /, /)
358
+ wrap(hash_str(v), sub_stat_value_width, sub_value_indent, ", ")
389
359
  end
390
360
  else
391
361
  "#{v || "none"}"
@@ -457,19 +427,20 @@ module RightSupport
457
427
  # "recent"(Array):: Most recent as a hash of "count", "type", "message", "when", and "where"
458
428
  # indent(String):: Indentation for each line
459
429
  # options(Hash):: Formatting options
460
- # :sub_stat_value_width(Integer):: Maximum characters in displayed sub-stat value line
430
+ # :exception_message_width(Integer):: Maximum characters displayed for exception message
461
431
  #
462
432
  # === Return
463
433
  # (String):: Exceptions in displayable format with line separators
464
434
  def self.exceptions_str(exceptions, indent, options = {})
465
- sub_stat_value_width = options[:sub_stat_value_width] || MAX_SUB_STAT_VALUE_WIDTH
435
+ exception_message_width = options[:exception_message_width] || MAX_EXCEPTION_MESSAGE_WIDTH
466
436
  indent2 = indent + (" " * 4)
467
437
  exceptions.to_a.sort.map do |k, v|
468
438
  sprintf("%s total: %d, most recent:\n", k, v["total"]) + v["recent"].reverse.map do |e|
469
- where = " IN #{e["where"]}" if e["where"]
470
- indent + wrap("(#{e["count"]}) #{time_at(e["when"])} #{e["type"]}: #{e["message"]}#{where}",
471
- [sub_stat_value_width, sub_stat_value_width + indent2.size],
472
- indent2, / |\/\/|\/|::|\.|-/)
439
+ message = e["message"]
440
+ if message && message.size > (exception_message_width - 3)
441
+ message = e["message"][0, exception_message_width - 3] + "..."
442
+ end
443
+ indent + "(#{e["count"]}) #{time_at(e["when"])} #{e["type"]}: #{message}\n" + indent2 + "#{e["where"]}"
473
444
  end.join("\n")
474
445
  end.join("\n" + indent)
475
446
  end