rails_pulse 0.4.2 → 0.5.0.pre.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +33 -1
- data/README.md +15 -4
- data/app/assets/stylesheets/rails_pulse/components/chart.css +11 -0
- data/app/assets/stylesheets/rails_pulse/components/descriptive_list.css +19 -1
- data/app/assets/stylesheets/rails_pulse/components/row.css +3 -17
- data/app/controllers/concerns/chart_table_concern.rb +28 -69
- data/app/controllers/concerns/deployment_markers_concern.rb +3 -1
- data/app/controllers/concerns/metric_card_concern.rb +6 -7
- data/app/controllers/rails_pulse/api/v1/base_controller.rb +130 -0
- data/app/controllers/rails_pulse/api/v1/capabilities_controller.rb +27 -0
- data/app/controllers/rails_pulse/api/v1/coverage_controller.rb +158 -0
- data/app/controllers/rails_pulse/api/v1/deployments_controller.rb +24 -0
- data/app/controllers/rails_pulse/api/v1/exceptions_controller.rb +76 -0
- data/app/controllers/rails_pulse/api/v1/job_runs_controller.rb +34 -0
- data/app/controllers/rails_pulse/api/v1/jobs_controller.rb +172 -0
- data/app/controllers/rails_pulse/api/v1/queries_controller.rb +114 -0
- data/app/controllers/rails_pulse/api/v1/requests_controller.rb +58 -0
- data/app/controllers/rails_pulse/api/v1/routes_controller.rb +98 -0
- data/app/controllers/rails_pulse/application_controller.rb +1 -1
- data/app/controllers/rails_pulse/dashboard_controller.rb +20 -20
- data/app/controllers/rails_pulse/deployments_controller.rb +23 -13
- data/app/controllers/rails_pulse/exceptions_controller.rb +10 -12
- data/app/controllers/rails_pulse/jobs_controller.rb +1 -2
- data/app/controllers/rails_pulse/queries_controller.rb +6 -5
- data/app/controllers/rails_pulse/requests_controller.rb +15 -14
- data/app/controllers/rails_pulse/routes_controller.rb +4 -8
- data/app/helpers/rails_pulse/application_helper.rb +15 -0
- data/app/helpers/rails_pulse/chart_helper.rb +9 -2
- data/app/helpers/rails_pulse/formatting_helper.rb +11 -12
- data/app/javascript/rails_pulse/controllers/chart_controller.js +80 -43
- data/app/jobs/rails_pulse/backfill_summaries_job.rb +4 -2
- data/app/models/concerns/rails_pulse/has_metadata.rb +15 -0
- data/app/models/concerns/rails_pulse/taggable.rb +5 -12
- data/app/models/rails_pulse/cards/base.rb +20 -11
- data/app/models/rails_pulse/charts/base.rb +2 -6
- data/app/models/rails_pulse/charts/percentile_chart_base.rb +3 -4
- data/app/models/rails_pulse/dashboard/charts/response_time_percentiles.rb +4 -5
- data/app/models/rails_pulse/dashboard/charts/throughput_and_errors.rb +4 -5
- data/app/models/rails_pulse/dashboard/concerns/time_range_helper.rb +1 -2
- data/app/models/rails_pulse/dashboard/health_summary.rb +30 -3
- data/app/models/rails_pulse/dashboard/needs_attention.rb +2 -3
- data/app/models/rails_pulse/dashboard/storage_pressure.rb +34 -4
- data/app/models/rails_pulse/dashboard/storage_status.rb +36 -0
- data/app/models/rails_pulse/deployment.rb +2 -7
- data/app/models/rails_pulse/event.rb +40 -0
- data/app/models/rails_pulse/exception_occurrence.rb +1 -1
- data/app/models/rails_pulse/exceptions/cards/exception_rate.rb +3 -6
- data/app/models/rails_pulse/exceptions/cards/open_groups.rb +1 -4
- data/app/models/rails_pulse/exceptions/cards/total_occurrences.rb +3 -6
- data/app/models/rails_pulse/exceptions/charts/occurrence_volume.rb +4 -5
- data/app/models/rails_pulse/job_run.rb +1 -1
- data/app/models/rails_pulse/jobs/cards/failure_rate.rb +8 -14
- data/app/models/rails_pulse/jobs/cards/p95_duration.rb +8 -14
- data/app/models/rails_pulse/jobs/cards/total_runs.rb +5 -11
- data/app/models/rails_pulse/jobs/charts/duration.rb +1 -1
- data/app/models/rails_pulse/jobs/charts/execution_volume.rb +1 -1
- data/app/models/rails_pulse/jobs/charts/failure_rate.rb +1 -1
- data/app/models/rails_pulse/like_pattern.rb +29 -0
- data/app/models/rails_pulse/queries/cards/average_query_times.rb +6 -12
- data/app/models/rails_pulse/queries/cards/database_load.rb +0 -9
- data/app/models/rails_pulse/queries/cards/execution_rate.rb +5 -11
- data/app/models/rails_pulse/queries/cards/percentile_query_times.rb +6 -12
- data/app/models/rails_pulse/queries/charts/database_load.rb +5 -6
- data/app/models/rails_pulse/queries/charts/execution_volume.rb +1 -1
- data/app/models/rails_pulse/request.rb +1 -1
- data/app/models/rails_pulse/routes/cards/error_rates.rb +5 -15
- data/app/models/rails_pulse/routes/cards/percentile_response_times.rb +5 -17
- data/app/models/rails_pulse/routes/cards/request_count_totals.rb +5 -17
- data/app/models/rails_pulse/routes/charts/error_rate.rb +1 -1
- data/app/models/rails_pulse/routes/charts/request_volume.rb +1 -1
- data/app/models/rails_pulse/summary.rb +5 -0
- data/app/models/rails_pulse/tables/base.rb +5 -6
- data/app/models/rails_pulse/time_range.rb +309 -0
- data/app/models/rails_pulse/writer_heartbeat.rb +86 -0
- data/app/serializers/rails_pulse/api/v1/deployment_serializer.rb +20 -0
- data/app/serializers/rails_pulse/api/v1/exception_group_serializer.rb +23 -0
- data/app/serializers/rails_pulse/api/v1/exception_occurrence_serializer.rb +22 -0
- data/app/serializers/rails_pulse/api/v1/job_run_serializer.rb +28 -0
- data/app/serializers/rails_pulse/api/v1/job_serializer.rb +26 -0
- data/app/serializers/rails_pulse/api/v1/query_serializer.rb +25 -0
- data/app/serializers/rails_pulse/api/v1/request_serializer.rb +21 -0
- data/app/serializers/rails_pulse/api/v1/route_serializer.rb +19 -0
- data/app/services/rails_pulse/exception_capture_service.rb +1 -1
- data/app/services/rails_pulse/operations/comparison.rb +1 -1
- data/app/services/rails_pulse/summary_service/from_child_periods.rb +161 -0
- data/app/services/rails_pulse/summary_service/from_raw_rows.rb +115 -0
- data/app/services/rails_pulse/summary_service/metrics.rb +38 -0
- data/app/services/rails_pulse/summary_service.rb +121 -133
- data/app/services/rails_pulse/tag_filter_service.rb +5 -5
- data/app/views/layouts/rails_pulse/_time_range_selector.html.erb +4 -1
- data/app/views/rails_pulse/components/_operation_details_popover.html.erb +1 -1
- data/app/views/rails_pulse/components/_table_head.html.erb +2 -2
- data/app/views/rails_pulse/dashboard/_chart_panel.html.erb +6 -1
- data/app/views/rails_pulse/dashboard/_deployments_panel.html.erb +2 -2
- data/app/views/rails_pulse/dashboard/_health_badge.html.erb +29 -20
- data/app/views/rails_pulse/dashboard/index.html.erb +16 -2
- data/app/views/rails_pulse/jobs/_chart_tabs.html.erb +1 -0
- data/app/views/rails_pulse/jobs/index.html.erb +1 -1
- data/app/views/rails_pulse/queries/_chart_tabs.html.erb +1 -0
- data/app/views/rails_pulse/queries/index.html.erb +1 -1
- data/app/views/rails_pulse/queries/show.html.erb +1 -1
- data/app/views/rails_pulse/requests/index.html.erb +1 -1
- data/app/views/rails_pulse/routes/_chart_tabs.html.erb +1 -0
- data/app/views/rails_pulse/routes/index.html.erb +1 -1
- data/app/views/rails_pulse/routes/show.html.erb +1 -1
- data/app/views/rails_pulse/shared/_aggregation_zone_badge.html.erb +4 -0
- data/app/views/rails_pulse/storage/show.html.erb +37 -0
- data/config/routes.rb +19 -0
- data/db/rails_pulse_migrate/20260506000001_create_rails_pulse_exceptions.rb +1 -1
- data/db/rails_pulse_migrate/20260921000001_create_rails_pulse_events.rb +21 -0
- data/db/rails_pulse_schema.rb +20 -2
- data/exe/rails-pulse +7 -0
- data/lib/generators/rails_pulse/base_methods.rb +1 -0
- data/lib/generators/rails_pulse/templates/db/rails_pulse_schema.rb +20 -2
- data/lib/generators/rails_pulse/templates/rails_pulse.rb +36 -13
- data/lib/rails_pulse/cleanup_service.rb +26 -20
- data/lib/rails_pulse/cli/agent_files/agents.md +87 -0
- data/lib/rails_pulse/cli/agent_files/claude_skill.md +165 -0
- data/lib/rails_pulse/cli/base_command.rb +37 -0
- data/lib/rails_pulse/cli/client.rb +64 -0
- data/lib/rails_pulse/cli/config.rb +87 -0
- data/lib/rails_pulse/cli/configure.rb +75 -0
- data/lib/rails_pulse/cli/coverage.rb +91 -0
- data/lib/rails_pulse/cli/deployments.rb +40 -0
- data/lib/rails_pulse/cli/exceptions.rb +111 -0
- data/lib/rails_pulse/cli/formatter.rb +31 -0
- data/lib/rails_pulse/cli/install.rb +145 -0
- data/lib/rails_pulse/cli/job_runs.rb +53 -0
- data/lib/rails_pulse/cli/jobs.rb +71 -0
- data/lib/rails_pulse/cli/main.rb +72 -0
- data/lib/rails_pulse/cli/mcp.rb +47 -0
- data/lib/rails_pulse/cli/queries.rb +62 -0
- data/lib/rails_pulse/cli/requests.rb +49 -0
- data/lib/rails_pulse/cli/routes.rb +66 -0
- data/lib/rails_pulse/configuration.rb +23 -1
- data/lib/rails_pulse/current.rb +11 -0
- data/lib/rails_pulse/engine.rb +12 -3
- data/lib/rails_pulse/job_run_collector.rb +15 -15
- data/lib/rails_pulse/mcp/server.rb +76 -0
- data/lib/rails_pulse/mcp/tools/coverage.rb +96 -0
- data/lib/rails_pulse/mcp/tools/deployments.rb +80 -0
- data/lib/rails_pulse/mcp/tools/endpoint.rb +167 -0
- data/lib/rails_pulse/mcp/tools/errors.rb +95 -0
- data/lib/rails_pulse/mcp/tools/exception_detail.rb +118 -0
- data/lib/rails_pulse/mcp/tools/exceptions.rb +114 -0
- data/lib/rails_pulse/mcp/tools/helpers.rb +162 -0
- data/lib/rails_pulse/mcp/tools/jobs.rb +155 -0
- data/lib/rails_pulse/mcp/tools/queries.rb +136 -0
- data/lib/rails_pulse/mcp/tools/routes.rb +97 -0
- data/lib/rails_pulse/mcp/tools/slow_requests.rb +118 -0
- data/lib/rails_pulse/middleware/request_collector.rb +9 -9
- data/lib/rails_pulse/scoped_inflector.rb +35 -0
- data/lib/rails_pulse/statistics.rb +27 -0
- data/lib/rails_pulse/subscribers/exception_subscriber.rb +2 -2
- data/lib/rails_pulse/subscribers/operation_subscriber.rb +15 -29
- data/lib/rails_pulse/tasks/status_reporter.rb +26 -0
- data/lib/rails_pulse/tracker.rb +75 -14
- data/lib/rails_pulse/version.rb +1 -1
- data/lib/rails_pulse.rb +0 -4
- data/public/rails-pulse-assets/rails-pulse.css +1 -1
- data/public/rails-pulse-assets/rails-pulse.js +13 -13
- metadata +96 -24
- data/app/controllers/concerns/page_timings.rb +0 -10
- data/app/controllers/concerns/response_range_concern.rb +0 -45
- data/app/controllers/concerns/time_range_concern.rb +0 -152
- data/app/controllers/concerns/zoom_range_concern.rb +0 -86
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
module RailsPulse
|
|
2
|
+
module Api
|
|
3
|
+
module V1
|
|
4
|
+
# What Rails Pulse knows, and how recently, so a caller can tell "nothing
|
|
5
|
+
# went wrong" from "nothing was recorded". Every other endpoint answers a
|
|
6
|
+
# question about the data; this one answers whether the data is there.
|
|
7
|
+
class CoverageController < BaseController
|
|
8
|
+
# A writer silent for longer than this is treated as not collecting,
|
|
9
|
+
# matching the window the dashboard and status task use.
|
|
10
|
+
LIVE_WINDOW = RailsPulse::WriterHeartbeat::LIVE_WINDOW
|
|
11
|
+
|
|
12
|
+
# Hourly summaries fall behind by up to an hour in normal operation,
|
|
13
|
+
# so staleness is measured past that.
|
|
14
|
+
SUMMARY_GRACE = 2.hours
|
|
15
|
+
|
|
16
|
+
def show
|
|
17
|
+
render json: {
|
|
18
|
+
as_of: Time.current.utc.iso8601,
|
|
19
|
+
telemetry: telemetry,
|
|
20
|
+
summaries: summaries,
|
|
21
|
+
retention: retention,
|
|
22
|
+
collection: collection
|
|
23
|
+
}
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
# The newest and oldest record of each kind. An empty pair means
|
|
29
|
+
# nothing of that kind has been recorded — not that nothing happened.
|
|
30
|
+
def telemetry
|
|
31
|
+
config = RailsPulse.configuration
|
|
32
|
+
{
|
|
33
|
+
requests: config.enabled ? span(RailsPulse::Request, :occurred_at) : untracked("config.enabled is false"),
|
|
34
|
+
job_runs: config.track_jobs ? span(RailsPulse::JobRun, :occurred_at) : untracked("config.track_jobs is false"),
|
|
35
|
+
exceptions: exception_span
|
|
36
|
+
}
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def untracked(reason)
|
|
40
|
+
{ tracked: false, reason: reason }
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def span(model, column)
|
|
44
|
+
oldest = model.minimum(column)
|
|
45
|
+
newest = model.maximum(column)
|
|
46
|
+
{
|
|
47
|
+
oldest: oldest&.utc&.iso8601,
|
|
48
|
+
newest: newest&.utc&.iso8601,
|
|
49
|
+
count: model.count,
|
|
50
|
+
tracked: true
|
|
51
|
+
}
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def exception_span
|
|
55
|
+
return untracked("config.track_exceptions is false") unless RailsPulse.configuration.track_exceptions
|
|
56
|
+
return untracked("the exception tables are missing; run the upgrade generator") unless RailsPulse::ExceptionOccurrence.table_exists?
|
|
57
|
+
|
|
58
|
+
span(RailsPulse::ExceptionOccurrence, :occurred_at)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Aggregated data outlives raw records, so a question about a window
|
|
62
|
+
# older than raw retention is answered from here or not at all.
|
|
63
|
+
def summaries
|
|
64
|
+
latest = RailsPulse::Summary.overall_requests.for_period_type("hour").maximum(:period_end)
|
|
65
|
+
earliest = RailsPulse::Summary.overall_requests.for_period_type("hour").minimum(:period_start)
|
|
66
|
+
|
|
67
|
+
{
|
|
68
|
+
hourly_from: earliest&.utc&.iso8601,
|
|
69
|
+
hourly_through: latest&.utc&.iso8601,
|
|
70
|
+
stale: latest.nil? || (Time.current - latest) > SUMMARY_GRACE,
|
|
71
|
+
note: summary_note(latest)
|
|
72
|
+
}
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def summary_note(latest)
|
|
76
|
+
return "RailsPulse::SummaryJob has never run, so no window can be answered from summaries." if latest.nil?
|
|
77
|
+
|
|
78
|
+
age_hours = ((Time.current - latest) / 1.hour).round
|
|
79
|
+
return nil if age_hours <= (SUMMARY_GRACE / 1.hour)
|
|
80
|
+
|
|
81
|
+
"RailsPulse::SummaryJob is about #{age_hours}h behind, so recent windows are incomplete."
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# What the configuration will keep, which bounds every question that
|
|
85
|
+
# can still be asked.
|
|
86
|
+
def retention
|
|
87
|
+
config = RailsPulse.configuration
|
|
88
|
+
{
|
|
89
|
+
raw_records: duration_label(config.full_retention_period),
|
|
90
|
+
hourly_summaries: duration_label(config.hourly_summary_retention),
|
|
91
|
+
events: duration_label(config.event_retention_period),
|
|
92
|
+
archiving_enabled: config.archiving_enabled
|
|
93
|
+
}
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def duration_label(value)
|
|
97
|
+
return nil if value.nil?
|
|
98
|
+
|
|
99
|
+
seconds = value.respond_to?(:to_i) ? value.to_i : nil
|
|
100
|
+
seconds ? { seconds: seconds, days: (seconds / 86_400.0).round(2) } : nil
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Gaps: requests a writer dropped because its queue was full, which
|
|
104
|
+
# means the numbers understate what happened. A writer that has gone
|
|
105
|
+
# quiet is not one: it starts with a process's first tracked request
|
|
106
|
+
# and a new one starts on the next, so a missing heartbeat means
|
|
107
|
+
# nothing has been queued, not that something was lost.
|
|
108
|
+
def collection
|
|
109
|
+
unless RailsPulse.configuration.async
|
|
110
|
+
return {
|
|
111
|
+
known: false,
|
|
112
|
+
gap_suspected: false,
|
|
113
|
+
reason: "config.async is false, so requests are written inline: nothing is queued or dropped, " \
|
|
114
|
+
"and no writer heartbeat is recorded"
|
|
115
|
+
}
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
unless RailsPulse::Event.table_available?
|
|
119
|
+
return { known: false, reason: "the events table is missing, so writer heartbeats are not recorded" }
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
stats = RailsPulse::WriterHeartbeat.summary(window: 1.hour)
|
|
123
|
+
last_seen = stats[:last_sampled_at]
|
|
124
|
+
|
|
125
|
+
{
|
|
126
|
+
known: true,
|
|
127
|
+
live_writers: stats[:processes],
|
|
128
|
+
queue_depth: stats[:queue_depth],
|
|
129
|
+
queue_size: stats[:queue_size],
|
|
130
|
+
dropped_last_hour: stats[:dropped],
|
|
131
|
+
last_heartbeat_at: last_seen&.utc&.iso8601,
|
|
132
|
+
gap_suspected: gap_suspected?(stats, last_seen),
|
|
133
|
+
note: collection_note(stats, last_seen)
|
|
134
|
+
}
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def gap_suspected?(stats, _last_seen)
|
|
138
|
+
stats[:dropped].positive?
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def collection_note(stats, last_seen)
|
|
142
|
+
notes = []
|
|
143
|
+
if last_seen.nil?
|
|
144
|
+
notes << "No writer heartbeat is on record. A writer starts with a process's first tracked request, " \
|
|
145
|
+
"so the app has had no tracked web traffic, or its web processes are not running."
|
|
146
|
+
elsif (Time.current - last_seen) > LIVE_WINDOW
|
|
147
|
+
notes << "No writer has reported since #{last_seen.utc.iso8601}, so no request has been queued since " \
|
|
148
|
+
"then: the app has had no tracked web traffic, or its web processes are not running."
|
|
149
|
+
end
|
|
150
|
+
if stats[:dropped].positive?
|
|
151
|
+
notes << "#{stats[:dropped]} request(s) were dropped in the last hour because the writer queue was full, so counts understate traffic."
|
|
152
|
+
end
|
|
153
|
+
notes.any? ? notes.join(" ") : nil
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
module RailsPulse
|
|
2
|
+
module Api
|
|
3
|
+
module V1
|
|
4
|
+
class DeploymentsController < BaseController
|
|
5
|
+
def index
|
|
6
|
+
parsed_range = time_range
|
|
7
|
+
return unless parsed_range
|
|
8
|
+
since_start, until_end = parsed_range
|
|
9
|
+
|
|
10
|
+
collection = RailsPulse::Deployment.recent.order(id: :desc)
|
|
11
|
+
collection = collection.where(started_at: since_start..) if since_start
|
|
12
|
+
collection = collection.where(started_at: ..until_end) if until_end
|
|
13
|
+
|
|
14
|
+
data, meta = paginated(collection)
|
|
15
|
+
|
|
16
|
+
render json: {
|
|
17
|
+
data: data.map { |deployment| DeploymentSerializer.serialize(deployment) },
|
|
18
|
+
meta: meta
|
|
19
|
+
}
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
module RailsPulse
|
|
2
|
+
module Api
|
|
3
|
+
module V1
|
|
4
|
+
# Exception groups: one row per distinct exception class and location,
|
|
5
|
+
# with its lifecycle status and how often and how recently it fired.
|
|
6
|
+
# This is what an agent or a desktop notifier reads to answer "what is
|
|
7
|
+
# broken right now?" — the raw 5xx request list under /requests cannot
|
|
8
|
+
# say which exception it was, only that the response failed.
|
|
9
|
+
class ExceptionsController < BaseController
|
|
10
|
+
STATUSES = RailsPulse::ExceptionGroup::STATUSES
|
|
11
|
+
SORT_COLUMNS = %w[last_seen_at first_seen_at occurrence_count].freeze
|
|
12
|
+
DEFAULT_OCCURRENCES = 5
|
|
13
|
+
MAX_OCCURRENCES = 20
|
|
14
|
+
|
|
15
|
+
rescue_from ActiveRecord::RecordNotFound do
|
|
16
|
+
render json: { error: "No exception group with id #{params[:id]}" }, status: :not_found
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def index
|
|
20
|
+
parsed_range = time_range
|
|
21
|
+
return unless parsed_range
|
|
22
|
+
since_start, until_end = parsed_range
|
|
23
|
+
|
|
24
|
+
status = params[:status].presence
|
|
25
|
+
if status && !STATUSES.include?(status)
|
|
26
|
+
return render json: { error: "Invalid status. Valid values: #{STATUSES.join(', ')}" }, status: :bad_request
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
sort = params[:sort].presence || "last_seen_at"
|
|
30
|
+
unless SORT_COLUMNS.include?(sort)
|
|
31
|
+
return render json: { error: "Invalid sort. Valid values: #{SORT_COLUMNS.join(', ')}" }, status: :bad_request
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
collection = RailsPulse::ExceptionGroup.order(sort => :desc, id: :desc)
|
|
35
|
+
collection = collection.where(status: status) if status
|
|
36
|
+
collection = collection.where(last_seen_at: since_start..) if since_start
|
|
37
|
+
collection = collection.where(last_seen_at: ..until_end) if until_end
|
|
38
|
+
collection = apply_search(collection)
|
|
39
|
+
|
|
40
|
+
data, meta = paginated(collection)
|
|
41
|
+
render json: { data: data.map { |group| ExceptionGroupSerializer.serialize(group) }, meta: meta }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# One group with its most recent occurrences: backtrace, request and
|
|
45
|
+
# params for each. This is what an agent reads to find the cause,
|
|
46
|
+
# so it carries everything the dashboard's exception page shows.
|
|
47
|
+
def show
|
|
48
|
+
group = RailsPulse::ExceptionGroup.find(params[:id])
|
|
49
|
+
count = integer_param(:occurrences, DEFAULT_OCCURRENCES, 1..MAX_OCCURRENCES)
|
|
50
|
+
occurrences = group.occurrences.order(occurred_at: :desc, id: :desc).limit(count)
|
|
51
|
+
|
|
52
|
+
render json: {
|
|
53
|
+
data: ExceptionGroupSerializer.serialize(group).merge(
|
|
54
|
+
occurrences: occurrences.map { |occurrence| ExceptionOccurrenceSerializer.serialize(occurrence) }
|
|
55
|
+
)
|
|
56
|
+
}
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
private
|
|
60
|
+
|
|
61
|
+
# Case-insensitive substring match on the exception class or the
|
|
62
|
+
# app-code location it was raised from.
|
|
63
|
+
def apply_search(scope)
|
|
64
|
+
return scope unless params[:search].present?
|
|
65
|
+
|
|
66
|
+
term = RailsPulse::LikePattern.containing(params[:search].to_s.downcase)
|
|
67
|
+
scope.where(
|
|
68
|
+
"LOWER(exception_class) LIKE :term #{RailsPulse::LikePattern::CLAUSE} " \
|
|
69
|
+
"OR LOWER(location) LIKE :term #{RailsPulse::LikePattern::CLAUSE}",
|
|
70
|
+
term: term
|
|
71
|
+
)
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
module RailsPulse
|
|
2
|
+
module Api
|
|
3
|
+
module V1
|
|
4
|
+
class JobRunsController < BaseController
|
|
5
|
+
def index
|
|
6
|
+
parsed_range = time_range
|
|
7
|
+
return unless parsed_range
|
|
8
|
+
since_start, until_end = parsed_range
|
|
9
|
+
|
|
10
|
+
collection = RailsPulse::JobRun.includes(:job).order(occurred_at: :desc, id: :desc)
|
|
11
|
+
collection = collection.where(occurred_at: since_start..) if since_start
|
|
12
|
+
collection = collection.where(occurred_at: ..until_end) if until_end
|
|
13
|
+
|
|
14
|
+
status = params[:status].presence
|
|
15
|
+
if status == "failed"
|
|
16
|
+
collection = collection.failed
|
|
17
|
+
elsif status
|
|
18
|
+
unless RailsPulse::JobRun::STATUSES.include?(status)
|
|
19
|
+
return render json: { error: "Invalid status. Valid values: #{RailsPulse::JobRun::STATUSES.join(', ')}" }, status: :bad_request
|
|
20
|
+
end
|
|
21
|
+
collection = collection.where(status: status)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
if params[:job].present?
|
|
25
|
+
collection = collection.joins(:job).where(rails_pulse_jobs: { name: params[:job] })
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
data, meta = paginated(collection)
|
|
29
|
+
render json: { data: data.map { |run| JobRunSerializer.serialize(run) }, meta: meta }
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
module RailsPulse
|
|
2
|
+
module Api
|
|
3
|
+
module V1
|
|
4
|
+
class JobsController < BaseController
|
|
5
|
+
STATUSES = %w[failed].freeze
|
|
6
|
+
|
|
7
|
+
def index
|
|
8
|
+
parsed_range = time_range
|
|
9
|
+
return unless parsed_range
|
|
10
|
+
since_start, until_end = parsed_range
|
|
11
|
+
|
|
12
|
+
collection = RailsPulse::Job.all.order(:name, :id)
|
|
13
|
+
collection = collection.where(name: params[:job]) if params[:job].present?
|
|
14
|
+
|
|
15
|
+
status = params[:status].presence
|
|
16
|
+
if status && !STATUSES.include?(status)
|
|
17
|
+
return render json: { error: "Invalid status. Valid values: #{STATUSES.join(', ')}" }, status: :bad_request
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
return render_windowed(collection, since_start, until_end, failed_only: status == "failed") if since_start || until_end
|
|
21
|
+
|
|
22
|
+
collection = collection.with_failures if status == "failed"
|
|
23
|
+
data, meta = paginated(collection)
|
|
24
|
+
render json: { data: data.map { |job| JobSerializer.serialize(job) }, meta: meta }
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
# Job rows carry lifetime counters, so a windowed question is answered
|
|
30
|
+
# from the per-job summaries the summary service writes each period,
|
|
31
|
+
# plus the raw runs recorded since the last period it summarized.
|
|
32
|
+
# Returning the lifetime numbers instead would answer a different
|
|
33
|
+
# question without saying so.
|
|
34
|
+
def render_windowed(collection, since_start, until_end, failed_only:)
|
|
35
|
+
until_end ||= Time.current
|
|
36
|
+
since_start ||= until_end - 24.hours
|
|
37
|
+
|
|
38
|
+
# Hourly rows are pruned at hourly_summary_retention, so a window
|
|
39
|
+
# reaching past that is read from daily rows for its whole length:
|
|
40
|
+
# reading the hourly rows that survive would count only part of it.
|
|
41
|
+
period_type = since_start >= RailsPulse.configuration.hourly_summary_retention.ago ? "hour" : "day"
|
|
42
|
+
|
|
43
|
+
# Periods are bucketed in the application's time zone, so the window
|
|
44
|
+
# is widened to that zone's period boundaries, not UTC ones.
|
|
45
|
+
from = RailsPulse::Summary.normalize_period_start(period_type, since_start.in_time_zone)
|
|
46
|
+
summarized_through = summarized_through(period_type)
|
|
47
|
+
summary_to = summarized_through ? [ summarized_through, period_ceiling(period_type, until_end) ].min : from
|
|
48
|
+
summary_to = from if summary_to < from
|
|
49
|
+
|
|
50
|
+
# The runs after the last summarized period are not in any summary
|
|
51
|
+
# yet, so they are counted from the raw rows. Cleanup never deletes a
|
|
52
|
+
# run that has not been summarized, so they are all still there.
|
|
53
|
+
live_from = [ from, summary_to ].max
|
|
54
|
+
live = live_from < until_end ? live_rows(collection, live_from, until_end) : {}
|
|
55
|
+
summarized = summary_to > from ? summary_rows(collection, period_type, from, summary_to) : {}
|
|
56
|
+
|
|
57
|
+
jobs = RailsPulse::Job.where(id: summarized.keys | live.keys).index_by(&:id)
|
|
58
|
+
rows = jobs.values.map { |job| [ job, stats_for(summarized[job.id], live[job.id]) ] }
|
|
59
|
+
rows.select! { |_, stats| stats[:failures_count].positive? } if failed_only
|
|
60
|
+
rows.sort_by! { |job, _| [ job.name.to_s, job.id ] }
|
|
61
|
+
|
|
62
|
+
page = rows.drop(offset).first(limit)
|
|
63
|
+
|
|
64
|
+
window = { since: from.utc.iso8601, until: [ summary_to, until_end ].max.utc.iso8601, period_type: period_type }
|
|
65
|
+
window[:summarized_through] = summarized_through&.utc&.iso8601
|
|
66
|
+
window[:live_from] = live_from.utc.iso8601 if live_from < until_end
|
|
67
|
+
|
|
68
|
+
render json: {
|
|
69
|
+
data: page.map { |job, stats| JobSerializer.serialize(job, stats: stats) },
|
|
70
|
+
meta: { total: rows.size, limit: limit, offset: offset, window: window }
|
|
71
|
+
}
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# The end of the newest period SummaryJob has written. The overall
|
|
75
|
+
# request row is written every period even when nothing ran, so it
|
|
76
|
+
# advances on an app that records no requests between job runs.
|
|
77
|
+
def summarized_through(period_type)
|
|
78
|
+
latest = [
|
|
79
|
+
RailsPulse::Summary.overall_requests.for_period_type(period_type).maximum(:period_start),
|
|
80
|
+
RailsPulse::Summary.for_jobs.for_period_type(period_type).maximum(:period_start)
|
|
81
|
+
].compact.max
|
|
82
|
+
latest && advance(period_type, latest.in_time_zone)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def summary_rows(collection, period_type, from, to)
|
|
86
|
+
RailsPulse::Summary
|
|
87
|
+
.for_jobs
|
|
88
|
+
.where(period_type: period_type, period_start: from...to)
|
|
89
|
+
.where(summarizable_id: collection.select(:id))
|
|
90
|
+
.group(:summarizable_id)
|
|
91
|
+
.select(
|
|
92
|
+
"summarizable_id, COUNT(*) AS periods, SUM(count) AS runs, SUM(error_count) AS failures, " \
|
|
93
|
+
"SUM(total_duration) AS total_duration, MIN(min_duration) AS min_duration, " \
|
|
94
|
+
"MAX(max_duration) AS max_duration, MAX(p95_duration) AS p95_duration, " \
|
|
95
|
+
"MAX(p99_duration) AS p99_duration"
|
|
96
|
+
)
|
|
97
|
+
.index_by(&:summarizable_id)
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Counted the way SummaryService counts a period: finished runs only,
|
|
101
|
+
# and anything other than success is a failure.
|
|
102
|
+
def live_rows(collection, from, to)
|
|
103
|
+
failure = "CASE WHEN status = #{RailsPulse::JobRun.connection.quote('success')} THEN 0 ELSE 1 END"
|
|
104
|
+
|
|
105
|
+
RailsPulse::JobRun
|
|
106
|
+
.where(occurred_at: from...to, status: RailsPulse::JobRun::FINAL_STATUSES)
|
|
107
|
+
.where(job_id: collection.select(:id))
|
|
108
|
+
.group(:job_id)
|
|
109
|
+
.select(
|
|
110
|
+
"job_id, COUNT(*) AS runs, SUM(#{failure}) AS failures, SUM(duration) AS total_duration, " \
|
|
111
|
+
"MIN(duration) AS min_duration, MAX(duration) AS max_duration"
|
|
112
|
+
)
|
|
113
|
+
.index_by(&:job_id)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def stats_for(summarized, live)
|
|
117
|
+
sources = [ summarized, live ].compact
|
|
118
|
+
runs = sources.sum { |row| row.runs.to_i }
|
|
119
|
+
failures = sources.sum { |row| row.failures.to_i }
|
|
120
|
+
total_duration = sources.sum { |row| row.total_duration.to_f }
|
|
121
|
+
|
|
122
|
+
stats = {
|
|
123
|
+
runs_count: runs,
|
|
124
|
+
failures_count: failures,
|
|
125
|
+
failure_rate: runs > 0 ? ((failures.to_f / runs) * 100).round(2) : 0.0,
|
|
126
|
+
avg_duration: runs > 0 ? (total_duration / runs).round(2) : nil,
|
|
127
|
+
min_duration: sources.filter_map { |row| row.min_duration&.to_f }.min&.round(2),
|
|
128
|
+
max_duration: sources.filter_map { |row| row.max_duration&.to_f }.max&.round(2)
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
# A percentile is a property of a distribution, and the distribution
|
|
132
|
+
# behind each period is not kept, so percentiles from several periods
|
|
133
|
+
# cannot be combined. They are reported only when one summarized
|
|
134
|
+
# period holds every run in the window.
|
|
135
|
+
periods = summarized&.periods.to_i
|
|
136
|
+
if periods == 1 && live.nil?
|
|
137
|
+
stats[:p95_duration] = summarized.p95_duration&.to_f&.round(2)
|
|
138
|
+
stats[:p99_duration] = summarized.p99_duration&.to_f&.round(2)
|
|
139
|
+
else
|
|
140
|
+
stats[:percentiles_note] = percentiles_note(periods, live)
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
stats
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def percentiles_note(periods, live)
|
|
147
|
+
if live && periods.zero?
|
|
148
|
+
"Omitted: these runs have not been summarized yet, and percentiles are read from summaries. " \
|
|
149
|
+
"Request a window ending at or before meta.window.summarized_through for p95 and p99."
|
|
150
|
+
else
|
|
151
|
+
spans = live ? "#{periods} summary period(s) and runs not yet summarized" : "#{periods} summary periods"
|
|
152
|
+
"Omitted: the window spans #{spans}, which cannot be combined into one percentile. " \
|
|
153
|
+
"Request a window covering a single summarized period for p95 and p99."
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# The start of the period after the one `time` falls in, or `time`
|
|
158
|
+
# itself when it is already on a boundary.
|
|
159
|
+
def period_ceiling(period_type, time)
|
|
160
|
+
start = RailsPulse::Summary.normalize_period_start(period_type, time.in_time_zone)
|
|
161
|
+
start == time ? start : advance(period_type, start)
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# Calendar arithmetic in the application's zone, so a day is a day
|
|
165
|
+
# across a DST change.
|
|
166
|
+
def advance(period_type, time)
|
|
167
|
+
period_type == "hour" ? time + 1.hour : time + 1.day
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
end
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
module RailsPulse
|
|
2
|
+
module Api
|
|
3
|
+
module V1
|
|
4
|
+
class QueriesController < BaseController
|
|
5
|
+
SORT_COLUMNS = %w[total_duration avg_duration executions max_duration].freeze
|
|
6
|
+
|
|
7
|
+
def index
|
|
8
|
+
parsed_range = time_range
|
|
9
|
+
return unless parsed_range
|
|
10
|
+
since_start, until_end = parsed_range
|
|
11
|
+
|
|
12
|
+
sort = params[:sort].presence
|
|
13
|
+
if sort && !SORT_COLUMNS.include?(sort)
|
|
14
|
+
return render json: { error: "Invalid sort. Valid values: #{SORT_COLUMNS.join(', ')}" }, status: :bad_request
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# A route filter is answered from operations, which only the stats
|
|
18
|
+
# path reads, so it implies a window the same way a sort does.
|
|
19
|
+
if since_start || until_end || sort || params[:route].present?
|
|
20
|
+
since_start ||= 24.hours.ago if until_end.nil?
|
|
21
|
+
render_with_stats(since_start..until_end, sort || "total_duration")
|
|
22
|
+
else
|
|
23
|
+
data, meta = paginated(RailsPulse::Query.all.order(:id))
|
|
24
|
+
render json: { data: data.map { |query| QuerySerializer.serialize(query) }, meta: meta }
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def render_with_stats(range, sort)
|
|
31
|
+
base = RailsPulse::Operation.where.not(query_id: nil).where(occurred_at: range)
|
|
32
|
+
base = apply_route_filter(base)
|
|
33
|
+
total = base.distinct.count(:query_id)
|
|
34
|
+
|
|
35
|
+
rows = base
|
|
36
|
+
.group(:query_id)
|
|
37
|
+
# Qualified: the route filter joins requests, which has its own
|
|
38
|
+
# duration column.
|
|
39
|
+
.select(
|
|
40
|
+
"rails_pulse_operations.query_id, COUNT(*) AS executions, " \
|
|
41
|
+
"AVG(rails_pulse_operations.duration) AS avg_duration, " \
|
|
42
|
+
"MAX(rails_pulse_operations.duration) AS max_duration, " \
|
|
43
|
+
"SUM(rails_pulse_operations.duration) AS total_duration, " \
|
|
44
|
+
"MAX(rails_pulse_operations.repetition_count) AS max_repetition_count"
|
|
45
|
+
)
|
|
46
|
+
# query_id breaks ties so offset pages neither repeat nor skip rows.
|
|
47
|
+
.order(Arel.sql("#{sort} DESC, rails_pulse_operations.query_id ASC"))
|
|
48
|
+
.limit(limit)
|
|
49
|
+
.offset(offset)
|
|
50
|
+
.to_a
|
|
51
|
+
|
|
52
|
+
queries = RailsPulse::Query.where(id: rows.map(&:query_id)).index_by(&:id)
|
|
53
|
+
locations = source_locations(base, rows.map(&:query_id))
|
|
54
|
+
data = rows.filter_map do |row|
|
|
55
|
+
query = queries[row.query_id]
|
|
56
|
+
QuerySerializer.serialize(query, stats: stats_for(row, locations[row.query_id])) if query
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
render json: { data: data, meta: { total: total, limit: limit, offset: offset } }
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Restricts the operations to those issued while serving one route, so
|
|
63
|
+
# "what is slow inside this endpoint" is one call rather than a guess.
|
|
64
|
+
# A bare integer is a route id; anything else matches the controller
|
|
65
|
+
# action or path the way the routes endpoint's search does.
|
|
66
|
+
def apply_route_filter(scope)
|
|
67
|
+
route = params[:route].to_s
|
|
68
|
+
return scope if route.blank?
|
|
69
|
+
|
|
70
|
+
scope = scope.joins(request: :route)
|
|
71
|
+
return scope.where(rails_pulse_routes: { id: route.to_i }) if route.match?(/\A\d+\z/)
|
|
72
|
+
|
|
73
|
+
term = RailsPulse::LikePattern.containing(route.downcase)
|
|
74
|
+
scope.where(
|
|
75
|
+
"LOWER(rails_pulse_routes.controller_action) LIKE :term #{RailsPulse::LikePattern::CLAUSE} " \
|
|
76
|
+
"OR LOWER(rails_pulse_routes.path) LIKE :term #{RailsPulse::LikePattern::CLAUSE}",
|
|
77
|
+
term: term
|
|
78
|
+
)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Where each query was issued from, most frequent first. Without this a
|
|
82
|
+
# caller knows a query is slow but not which line of code runs it.
|
|
83
|
+
# Counted over the same operations as the stats, so with a route filter
|
|
84
|
+
# a query shared across the app names the call sites in that endpoint.
|
|
85
|
+
def source_locations(operations, query_ids, per_query: 3)
|
|
86
|
+
return {} if query_ids.empty?
|
|
87
|
+
|
|
88
|
+
counts = operations
|
|
89
|
+
.where(query_id: query_ids)
|
|
90
|
+
.where.not(codebase_location: nil)
|
|
91
|
+
.group("rails_pulse_operations.query_id", "rails_pulse_operations.codebase_location")
|
|
92
|
+
.count
|
|
93
|
+
|
|
94
|
+
counts.group_by { |(query_id, _), _| query_id }.transform_values do |entries|
|
|
95
|
+
entries.sort_by { |(_, location), count| [ -count, location ] }
|
|
96
|
+
.first(per_query)
|
|
97
|
+
.map { |(_, location), count| { location: location, count: count } }
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def stats_for(row, locations)
|
|
102
|
+
{
|
|
103
|
+
executions: row.executions.to_i,
|
|
104
|
+
avg_duration_ms: row.avg_duration.to_f.round(1),
|
|
105
|
+
max_duration_ms: row.max_duration.to_f.round(1),
|
|
106
|
+
total_duration_ms: row.total_duration.to_f.round(1),
|
|
107
|
+
max_repetition_count: row.max_repetition_count&.to_i,
|
|
108
|
+
source_locations: locations || []
|
|
109
|
+
}
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
module RailsPulse
|
|
2
|
+
module Api
|
|
3
|
+
module V1
|
|
4
|
+
class RequestsController < BaseController
|
|
5
|
+
STATUS_CLASS = /\A(\d)xx\z/
|
|
6
|
+
STATUS_CODE = /\A\d{3}\z/
|
|
7
|
+
|
|
8
|
+
def index
|
|
9
|
+
collection = RailsPulse::Request.all.order(occurred_at: :desc, id: :desc)
|
|
10
|
+
|
|
11
|
+
parsed_range = time_range
|
|
12
|
+
return unless parsed_range
|
|
13
|
+
since_start, until_end = parsed_range
|
|
14
|
+
|
|
15
|
+
collection = collection.where(occurred_at: since_start..) if since_start
|
|
16
|
+
collection = collection.where(occurred_at: ..until_end) if until_end
|
|
17
|
+
collection = apply_route_filter(collection)
|
|
18
|
+
collection = apply_status_filter(collection)
|
|
19
|
+
return if performed?
|
|
20
|
+
|
|
21
|
+
data, meta = paginated(collection)
|
|
22
|
+
render json: { data: data.map { |request| RequestSerializer.serialize(request) }, meta: meta }
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
private
|
|
26
|
+
|
|
27
|
+
# Case-insensitive substring match on the request's controller action
|
|
28
|
+
# or its route's path, so an agent can profile one endpoint without
|
|
29
|
+
# paging through every request in the window.
|
|
30
|
+
def apply_route_filter(scope)
|
|
31
|
+
return scope unless params[:route].present?
|
|
32
|
+
|
|
33
|
+
term = RailsPulse::LikePattern.containing(params[:route].to_s.downcase)
|
|
34
|
+
scope.joins(:route).where(
|
|
35
|
+
"LOWER(rails_pulse_requests.controller_action) LIKE :term #{RailsPulse::LikePattern::CLAUSE} " \
|
|
36
|
+
"OR LOWER(rails_pulse_routes.path) LIKE :term #{RailsPulse::LikePattern::CLAUSE}",
|
|
37
|
+
term: term
|
|
38
|
+
)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def apply_status_filter(scope)
|
|
42
|
+
return scope unless params[:status].present?
|
|
43
|
+
|
|
44
|
+
status = params[:status].to_s.downcase
|
|
45
|
+
if (match = status.match(STATUS_CLASS))
|
|
46
|
+
digit = match[1].to_i
|
|
47
|
+
scope.where(status: (digit * 100)...((digit + 1) * 100))
|
|
48
|
+
elsif status.match?(STATUS_CODE)
|
|
49
|
+
scope.where(status: status.to_i)
|
|
50
|
+
else
|
|
51
|
+
render json: { error: "Invalid status. Use a three-digit code (500) or a class (5xx)" }, status: :bad_request
|
|
52
|
+
scope
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|