rails_error_dashboard 0.14.4 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +12 -5
  3. data/app/controllers/rails_error_dashboard/application_controller.rb +28 -4
  4. data/app/controllers/rails_error_dashboard/errors_controller.rb +68 -2
  5. data/app/controllers/rails_error_dashboard/health_controller.rb +38 -0
  6. data/app/helpers/rails_error_dashboard/application_helper.rb +2 -2
  7. data/app/helpers/rails_error_dashboard/backtrace_helper.rb +56 -2
  8. data/app/jobs/rails_error_dashboard/baseline_alert_job.rb +20 -2
  9. data/app/jobs/rails_error_dashboard/cascade_detection_job.rb +30 -0
  10. data/app/jobs/rails_error_dashboard/concerns/plain_channel_message.rb +33 -6
  11. data/app/jobs/rails_error_dashboard/discord_error_notification_job.rb +1 -1
  12. data/app/jobs/rails_error_dashboard/pagerduty_error_notification_job.rb +1 -1
  13. data/app/jobs/rails_error_dashboard/retention_cleanup_job.rb +26 -0
  14. data/app/jobs/rails_error_dashboard/scheduled_digest_job.rb +6 -1
  15. data/app/jobs/rails_error_dashboard/slack_error_notification_job.rb +1 -1
  16. data/app/jobs/rails_error_dashboard/telegram_error_notification_job.rb +25 -0
  17. data/app/jobs/rails_error_dashboard/webhook_error_notification_job.rb +2 -18
  18. data/app/models/rails_error_dashboard/error_log.rb +43 -12
  19. data/app/models/rails_error_dashboard/missing_translation.rb +43 -0
  20. data/app/views/layouts/rails_error_dashboard.html.erb +11 -1
  21. data/app/views/rails_error_dashboard/errors/_error_info.html.erb +3 -1
  22. data/app/views/rails_error_dashboard/errors/_sidebar_metadata.html.erb +3 -1
  23. data/app/views/rails_error_dashboard/errors/_source_code.html.erb +3 -9
  24. data/app/views/rails_error_dashboard/errors/activestorage_health_summary.html.erb +28 -0
  25. data/app/views/rails_error_dashboard/errors/boot_errors.html.erb +130 -0
  26. data/app/views/rails_error_dashboard/errors/index.html.erb +1 -1
  27. data/app/views/rails_error_dashboard/errors/missing_translations.html.erb +143 -0
  28. data/app/views/rails_error_dashboard/errors/settings/_value_badge.html.erb +7 -4
  29. data/app/views/rails_error_dashboard/errors/settings.html.erb +19 -5
  30. data/config/locales/de.yml +68 -5
  31. data/config/locales/en.yml +78 -4
  32. data/config/locales/es.yml +68 -5
  33. data/config/locales/fr.yml +68 -5
  34. data/config/locales/it.yml +68 -5
  35. data/config/locales/ja.yml +68 -4
  36. data/config/locales/pl.yml +68 -5
  37. data/config/locales/pt-BR.yml +68 -5
  38. data/config/locales/ru.yml +68 -5
  39. data/config/locales/uk.yml +68 -5
  40. data/config/locales/zh-CN.yml +68 -4
  41. data/config/routes.rb +6 -0
  42. data/db/migrate/20251223000000_create_rails_error_dashboard_complete_schema.rb +10 -2
  43. data/db/migrate/20261010000001_create_missing_translations.rb +50 -0
  44. data/lib/generators/rails_error_dashboard/install/install_generator.rb +9 -4
  45. data/lib/generators/rails_error_dashboard/install/templates/initializer.rb +27 -0
  46. data/lib/rails_error_dashboard/commands/assign_error.rb +24 -1
  47. data/lib/rails_error_dashboard/commands/batch_resolve_errors.rb +9 -1
  48. data/lib/rails_error_dashboard/commands/flush_missing_translations.rb +103 -0
  49. data/lib/rails_error_dashboard/commands/log_error.rb +1 -1
  50. data/lib/rails_error_dashboard/commands/resolve_error.rb +2 -15
  51. data/lib/rails_error_dashboard/commands/update_error_status.rb +7 -0
  52. data/lib/rails_error_dashboard/configuration.rb +42 -1
  53. data/lib/rails_error_dashboard/engine.rb +42 -1
  54. data/lib/rails_error_dashboard/middleware/rate_limiter.rb +7 -1
  55. data/lib/rails_error_dashboard/plugin.rb +19 -3
  56. data/lib/rails_error_dashboard/plugin_registry.rb +61 -12
  57. data/lib/rails_error_dashboard/queries/boot_errors.rb +76 -0
  58. data/lib/rails_error_dashboard/queries/dashboard_stats.rb +3 -11
  59. data/lib/rails_error_dashboard/queries/errors_list.rb +8 -16
  60. data/lib/rails_error_dashboard/queries/health_status.rb +191 -0
  61. data/lib/rails_error_dashboard/queries/missing_translation_summary.rb +82 -0
  62. data/lib/rails_error_dashboard/services/active_storage_probe.rb +91 -0
  63. data/lib/rails_error_dashboard/services/backtrace_parser.rb +13 -7
  64. data/lib/rails_error_dashboard/services/backtrace_processor.rb +15 -0
  65. data/lib/rails_error_dashboard/services/baseline_alert_payload_builder.rb +30 -0
  66. data/lib/rails_error_dashboard/services/crash_capture.rb +85 -2
  67. data/lib/rails_error_dashboard/services/digest_builder.rb +9 -6
  68. data/lib/rails_error_dashboard/services/error_notification_dispatcher.rb +6 -0
  69. data/lib/rails_error_dashboard/services/github_link_generator.rb +25 -9
  70. data/lib/rails_error_dashboard/services/missing_translation_handler.rb +152 -0
  71. data/lib/rails_error_dashboard/services/missing_translation_tracker.rb +316 -0
  72. data/lib/rails_error_dashboard/services/resolution_callbacks.rb +44 -0
  73. data/lib/rails_error_dashboard/services/severity_classifier.rb +52 -0
  74. data/lib/rails_error_dashboard/services/source_code_reader.rb +23 -3
  75. data/lib/rails_error_dashboard/services/telegram_delivery.rb +134 -0
  76. data/lib/rails_error_dashboard/services/telegram_payload_builder.rb +105 -0
  77. data/lib/rails_error_dashboard/services/webhook_delivery.rb +63 -0
  78. data/lib/rails_error_dashboard/services/webhook_signer.rb +80 -0
  79. data/lib/rails_error_dashboard/version.rb +1 -1
  80. data/lib/rails_error_dashboard.rb +12 -0
  81. metadata +21 -2
@@ -14,6 +14,16 @@ module RailsErrorDashboard
14
14
  # disk as JSON. On the next boot, `import!` reads crash files and creates
15
15
  # ErrorLog records with severity "fatal".
16
16
  #
17
+ # BOOT CRASHES: the hook is registered from an engine initializer that
18
+ # runs right after the host's config/initializers (so the configuration
19
+ # is known) and before Rails' :eager_load! finisher. That ordering is the
20
+ # whole point: a Zeitwerk::NameError from eager loading, a SyntaxError in
21
+ # a model, an initializer that raised -- these kill the process before
22
+ # config.after_initialize ever runs, so a hook registered there (as this
23
+ # one was until 0.15) never saw them. A crash captured while
24
+ # Rails.application.initialized? is still false is marked phase "boot",
25
+ # imported with platform "boot_crash", and listed on the Boot Errors page.
26
+ #
17
27
  # Safety contract:
18
28
  # - Default OFF (opt-in via config.enable_crash_capture)
19
29
  # - Writes to tmpfile, NOT the database (connection pool may be closed)
@@ -24,6 +34,14 @@ module RailsErrorDashboard
24
34
  class CrashCapture
25
35
  FILE_PREFIX = "red_crash_"
26
36
 
37
+ BOOT_PHASE = "boot"
38
+ RUNTIME_PHASE = "runtime"
39
+ BOOT_PLATFORM = "boot_crash"
40
+ RUNTIME_PLATFORM = "crash_capture"
41
+
42
+ # "expected file /app/app/models/foo.rb to define constant Foo, but didn't"
43
+ ZEITWERK_MESSAGE = /expected file (?<file>.+?) to define constant (?<constant>\S+), but didn't/
44
+
27
45
  class << self
28
46
  # Enable crash capture. Registers the `at_exit` hook and records boot time.
29
47
  # @return [true]
@@ -108,6 +126,11 @@ module RailsErrorDashboard
108
126
  # Rails version (may not be available during crash)
109
127
  data[:rails_version] = Rails.version if defined?(Rails) && Rails.respond_to?(:version)
110
128
 
129
+ # Boot or runtime: a process that died before initialize! finished
130
+ # never served a request, which is a different kind of outage.
131
+ data[:phase] = boot_phase? ? BOOT_PHASE : RUNTIME_PHASE
132
+ data[:boot] = boot_detail(exception) if data[:phase] == BOOT_PHASE
133
+
111
134
  # Uptime
112
135
  if @boot_time
113
136
  data[:uptime_seconds] = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - @boot_time).round(1)
@@ -122,6 +145,51 @@ module RailsErrorDashboard
122
145
  data
123
146
  end
124
147
 
148
+ # True while Rails is still running its initializers (eager loading
149
+ # included). Rails.application.initialized? flips only after the last
150
+ # finisher has run. Outside Rails (a plain Ruby script) there is no
151
+ # boot phase to speak of.
152
+ def boot_phase?
153
+ defined?(Rails) && Rails.respond_to?(:application) && Rails.application &&
154
+ !Rails.application.initialized?
155
+ rescue
156
+ false
157
+ end
158
+
159
+ # What a boot crash names, for the Boot Errors page: the file and the
160
+ # constant for a Zeitwerk::NameError, else the first backtrace frame
161
+ # under Rails.root (a SyntaxError or an initializer that raised).
162
+ # Paths are made relative to Rails.root so the page reads
163
+ # "app/models/foo.rb", not a deploy directory.
164
+ def boot_detail(exception)
165
+ detail = {}
166
+ root = rails_root_prefix
167
+
168
+ if exception.class.name == "Zeitwerk::NameError" && (m = ZEITWERK_MESSAGE.match(exception.message.to_s))
169
+ detail[:file] = relative_path(m[:file], root)
170
+ detail[:constant] = m[:constant]
171
+ elsif (frame = Array(exception.backtrace).find { |f| root && f.to_s.start_with?(root) })
172
+ detail[:file] = relative_path(frame.to_s.split(":in ").first, root)
173
+ end
174
+
175
+ detail
176
+ rescue
177
+ {}
178
+ end
179
+
180
+ def rails_root_prefix
181
+ return nil unless defined?(Rails) && Rails.respond_to?(:root) && Rails.root
182
+
183
+ "#{Rails.root}/"
184
+ rescue
185
+ nil
186
+ end
187
+
188
+ def relative_path(path, root)
189
+ path = path.to_s
190
+ root && path.start_with?(root) ? path.delete_prefix(root) : path
191
+ end
192
+
125
193
  def extract_cause_chain(exception)
126
194
  causes = []
127
195
  current = exception.cause
@@ -155,6 +223,8 @@ module RailsErrorDashboard
155
223
  cause_chain = data["cause_chain"]
156
224
  cause_json = cause_chain.is_a?(Array) && cause_chain.any? ? cause_chain.to_json : nil
157
225
 
226
+ boot = data["phase"] == BOOT_PHASE
227
+
158
228
  # Build environment_info from crash metadata
159
229
  env_info = {
160
230
  ruby_version: data["ruby_version"],
@@ -164,6 +234,8 @@ module RailsErrorDashboard
164
234
  uptime_seconds: data["uptime_seconds"],
165
235
  gc: data["gc"],
166
236
  crash_captured_at: data["timestamp"],
237
+ phase: data["phase"],
238
+ boot: (data["boot"] if boot && data["boot"].is_a?(Hash) && data["boot"].any?),
167
239
  source: "crash_capture"
168
240
  }.compact
169
241
 
@@ -180,7 +252,9 @@ module RailsErrorDashboard
180
252
  message: data["message"] || "Process crash captured via at_exit hook",
181
253
  backtrace: backtrace_text,
182
254
  occurred_at: occurred_at,
183
- platform: "crash_capture",
255
+ # Boot crashes get their own platform so the Boot Errors page and
256
+ # the index's platform filter can find them.
257
+ platform: boot ? BOOT_PLATFORM : RUNTIME_PLATFORM,
184
258
  resolved: false
185
259
  }
186
260
 
@@ -210,7 +284,16 @@ module RailsErrorDashboard
210
284
  attributes[:last_seen_at] = occurred_at
211
285
  end
212
286
 
213
- ErrorLog.create!(attributes)
287
+ # Through the same find-or-increment as every other capture: a crash
288
+ # that repeats on every restart is one row with a count, not a new
289
+ # row per restart -- and a plain create! collided with the unique
290
+ # group-identity index on the second one, which left the file as
291
+ # .failed and the crash unrecorded.
292
+ if attributes[:error_hash]
293
+ Commands::FindOrIncrementError.call(attributes[:error_hash], attributes)
294
+ else
295
+ ErrorLog.create!(attributes)
296
+ end
214
297
 
215
298
  # Delete file after successful import
216
299
  File.delete(file)
@@ -93,9 +93,7 @@ module RailsErrorDashboard
93
93
  unresolved = scope.where(resolved: false).count
94
94
  # Severity is computed from error_type via SeverityClassifier (not a DB column).
95
95
  # Count critical+high by matching known error type patterns via SQL WHERE IN.
96
- critical_types = Services::SeverityClassifier::CRITICAL_ERROR_TYPES +
97
- Services::SeverityClassifier::HIGH_SEVERITY_ERROR_TYPES
98
- critical_high = scope.where(error_type: critical_types).count
96
+ critical_high = scope.where(error_type: critical_and_high_types).count
99
97
 
100
98
  total = resolved + unresolved
101
99
  resolution_rate = total > 0 ? (resolved.to_f / total * 100).round(1) : 0
@@ -141,11 +139,9 @@ module RailsErrorDashboard
141
139
  end
142
140
 
143
141
  def critical_unresolved
144
- critical_types = Services::SeverityClassifier::CRITICAL_ERROR_TYPES +
145
- Services::SeverityClassifier::HIGH_SEVERITY_ERROR_TYPES
146
142
  base_scope
147
143
  .where(resolved: false)
148
- .where(error_type: critical_types)
144
+ .where(error_type: critical_and_high_types)
149
145
  .order(occurred_at: :desc)
150
146
  .limit(5)
151
147
  .map do |error|
@@ -161,6 +157,13 @@ module RailsErrorDashboard
161
157
  []
162
158
  end
163
159
 
160
+
161
+ # Built-in lists adjusted by custom_severity_rules, so the digest's
162
+ # "critical/high" agrees with the dashboard's badges.
163
+ def critical_and_high_types
164
+ Services::SeverityClassifier.error_types_for(:critical) +
165
+ Services::SeverityClassifier.error_types_for(:high)
166
+ end
164
167
  def build_comparison
165
168
  previous_start = (@days * 2).days.ago
166
169
  previous_end = @start_date
@@ -19,6 +19,7 @@ module RailsErrorDashboard
19
19
  (config.enable_slack_notifications && config.slack_webhook_url.present?) ||
20
20
  (config.enable_email_notifications && config.notification_email_recipients.present?) ||
21
21
  (config.enable_discord_notifications && config.discord_webhook_url.present?) ||
22
+ TelegramDelivery.configured?(config) ||
22
23
  (config.enable_pagerduty_notifications && config.pagerduty_integration_key.present?) ||
23
24
  (config.enable_webhook_notifications && config.webhook_urls.present?) || false
24
25
  end
@@ -58,6 +59,11 @@ module RailsErrorDashboard
58
59
  fired << "discord"
59
60
  end
60
61
 
62
+ if TelegramDelivery.configured?(config)
63
+ TelegramErrorNotificationJob.perform_later(error_log.id, locale)
64
+ fired << "telegram"
65
+ end
66
+
61
67
  if config.enable_pagerduty_notifications && config.pagerduty_integration_key.present?
62
68
  PagerdutyErrorNotificationJob.perform_later(error_log.id, locale)
63
69
  fired << "pagerduty"
@@ -135,20 +135,36 @@ module RailsErrorDashboard
135
135
  def normalize_file_path
136
136
  path = file_path.strip
137
137
 
138
- # Remove leading slash
139
- path = path.sub(%r{^/}, "")
140
-
141
- # Remove Rails.root or app root prefix if present
142
- # Handles paths like "/Users/foo/myapp/app/models/user.rb" -> "app/models/user.rb"
143
- # Match pattern: look for one of the standard Rails directories
144
- match = path.match(%r{.*/?((?:app|lib|config|db|spec|test)/.*)$})
145
- if match
146
- path = match[1]
138
+ # A frame under Rails.root is that path relative to the root, exactly.
139
+ root = rails_root_prefix
140
+ path = path.delete_prefix(root) if root && path.start_with?(root)
141
+
142
+ if path.start_with?("/")
143
+ # An absolute path under some other root: the best guess is the
144
+ # last standard Rails directory in it ("/Users/foo/myapp/app/..." ->
145
+ # "app/...").
146
+ path = path.sub(%r{^/}, "")
147
+ match = path.match(%r{.*/((?:app|lib|config|db|spec|test)/.*)\z})
148
+ else
149
+ # Already relative (BacktraceProcessor stores frames that way): the
150
+ # FIRST standard directory at a segment boundary. The greedy guess
151
+ # above, applied here, linked app/services/config/loader.rb to
152
+ # config/loader.rb and app/models/contest/x.rb to test/x.rb.
153
+ match = path.match(%r{(?:\A|/)((?:app|lib|config|db|spec|test)/.*)\z})
147
154
  end
155
+ path = match[1] if match
148
156
 
149
157
  path
150
158
  end
151
159
 
160
+ def rails_root_prefix
161
+ return nil unless defined?(Rails) && Rails.respond_to?(:root) && Rails.root
162
+
163
+ "#{Rails.root}/"
164
+ rescue StandardError
165
+ nil
166
+ end
167
+
152
168
  # Generate GitHub link
153
169
  #
154
170
  # Format: https://github.com/user/repo/blob/{ref}/path/to/file.rb#L42
@@ -0,0 +1,152 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RailsErrorDashboard
4
+ module Services
5
+ # Wraps the host app's I18n.exception_handler so every missing translation
6
+ # is counted, then hands the exception to the handler the app already had.
7
+ #
8
+ # WHY THE EXCEPTION HANDLER: I18n.t calls it for every miss
9
+ # (i18n/lib/i18n.rb, handle_exception). There is no
10
+ # ActiveSupport::Notifications event for a miss, so subscribing, as the
11
+ # deprecation tracker does, is not an option.
12
+ #
13
+ # WHY ALSO A VIEW-HELPER HOOK: the view helper t() does NOT go through
14
+ # the exception handler before Rails 8.1. It looks the key up with a
15
+ # sentinel default (so I18n never misses) and renders the "translation
16
+ # missing" span from its own missing_translation method; only 8.1 added
17
+ # an exception_handler call there. Templates are where most misses live,
18
+ # so ViewHelperHook is prepended onto ActionView's TranslationHelper and
19
+ # counts in missing_translation itself. A thread flag set around super
20
+ # keeps the handler from counting the same miss a second time on 8.1.
21
+ #
22
+ # WHAT IS NOT SEEN, by design: a lookup given a :default never misses, and
23
+ # with raise_on_missing_translations (or `raise: true`) I18n raises before
24
+ # reaching any handler -- that miss surfaces as an exception, which RED
25
+ # captures as an error anyway.
26
+ #
27
+ # These are the only places RED touches global I18n and ActionView state,
28
+ # which is why they are opt-in (enable_missing_translation_tracking,
29
+ # default false) and transparent: no return value changes, no exception
30
+ # the original would have raised is swallowed, and uninstall! puts the
31
+ # original handler back (the view hook stays prepended -- Ruby cannot
32
+ # un-prepend -- but does nothing while the handler is not installed).
33
+ # RED's own dashboard strings never arrive here; I18nStore resolves them
34
+ # in a private backend.
35
+ class MissingTranslationHandler
36
+ # Set by ViewHelperHook around super so a Rails 8.1+ missing_translation,
37
+ # which calls the exception handler itself, is counted once, not twice.
38
+ VIEW_HOOK_FLAG = :red_missing_translation_in_view_hook
39
+
40
+ attr_reader :original
41
+
42
+ # @param original [#call, Symbol] the handler being wrapped. I18n allows
43
+ # a Symbol naming a method on I18n itself; it is delegated the same way
44
+ # I18n.handle_exception would.
45
+ def initialize(original)
46
+ @original = original
47
+ end
48
+
49
+ # The I18n exception handler contract.
50
+ def call(exception, locale, key, options)
51
+ track(exception, locale, key, options) if exception.is_a?(::I18n::MissingTranslation)
52
+ delegate(exception, locale, key, options)
53
+ end
54
+
55
+ # Prepended onto ActionView::Helpers::TranslationHelper: counts the miss
56
+ # the view helper is about to render as "translation missing".
57
+ module ViewHelperHook
58
+ private
59
+
60
+ def missing_translation(key, options)
61
+ if MissingTranslationHandler.installed?
62
+ MissingTranslationHandler.record_view_miss(key, options)
63
+ Thread.current.thread_variable_set(MissingTranslationHandler::VIEW_HOOK_FLAG, true)
64
+ end
65
+ super
66
+ ensure
67
+ Thread.current.thread_variable_set(MissingTranslationHandler::VIEW_HOOK_FLAG, nil)
68
+ end
69
+ end
70
+
71
+ class << self
72
+ # Wrap the current global handler and hook the view helper. Idempotent:
73
+ # a second install (code reloading, a spec that installs twice) does
74
+ # not stack wrappers.
75
+ # @return [MissingTranslationHandler]
76
+ def install!
77
+ install_view_hook!
78
+
79
+ current = ::I18n.exception_handler
80
+ return current if current.is_a?(self)
81
+
82
+ ::I18n.exception_handler = new(current)
83
+ end
84
+
85
+ # Put the wrapped handler back. A no-op when nothing is installed.
86
+ def uninstall!
87
+ current = ::I18n.exception_handler
88
+ return unless current.is_a?(self)
89
+
90
+ ::I18n.exception_handler = current.original
91
+ end
92
+
93
+ def installed?
94
+ ::I18n.exception_handler.is_a?(self)
95
+ end
96
+
97
+ # Count a miss the view helper reported. Never raises.
98
+ def record_view_miss(key, options)
99
+ options = {} unless options.is_a?(Hash)
100
+ locale = options[:locale] || ::I18n.locale
101
+ full_key = ::I18n.normalize_keys(nil, key, options[:scope]).join(".")
102
+
103
+ MissingTranslationTracker.record(locale: locale.to_s, key: full_key)
104
+ rescue => e
105
+ nil
106
+ end
107
+
108
+ private
109
+
110
+ def install_view_hook!
111
+ return unless defined?(::ActionView::Helpers::TranslationHelper)
112
+
113
+ helper = ::ActionView::Helpers::TranslationHelper
114
+ helper.prepend(ViewHelperHook) unless helper.ancestors.include?(ViewHelperHook)
115
+ rescue => e
116
+ RailsErrorDashboard::Logger.debug(
117
+ "[RailsErrorDashboard] MissingTranslationHandler view hook not installed: #{e.class} - #{e.message}"
118
+ )
119
+ nil
120
+ end
121
+ end
122
+
123
+ private
124
+
125
+ # Counting must never change what the host sees, so it is rescued on its
126
+ # own and the exception is delegated whatever happened here.
127
+ def track(exception, locale, key, options)
128
+ # The view hook already counted this one (Rails 8.1+ calls the
129
+ # handler from inside missing_translation).
130
+ return if Thread.current.thread_variable_get(VIEW_HOOK_FLAG)
131
+
132
+ # The full dotted key as I18n normalised it, scope included, locale
133
+ # excluded: "users.show.greeting", not "en.users.show.greeting".
134
+ scope = options.is_a?(Hash) ? options[:scope] : nil
135
+ full_key = ::I18n.normalize_keys(nil, exception.key, scope).join(".")
136
+
137
+ MissingTranslationTracker.record(locale: locale.to_s, key: full_key)
138
+ rescue => e
139
+ nil
140
+ end
141
+
142
+ def delegate(exception, locale, key, options)
143
+ case original
144
+ when Symbol
145
+ ::I18n.send(original, exception, locale, key, options)
146
+ else
147
+ original.call(exception, locale, key, options)
148
+ end
149
+ end
150
+ end
151
+ end
152
+ end