rails_pulse 0.4.2 → 0.5.0.pre.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +58 -1
- data/README.md +15 -4
- data/app/assets/stylesheets/rails_pulse/components/chart.css +11 -0
- data/app/assets/stylesheets/rails_pulse/components/descriptive_list.css +19 -1
- data/app/assets/stylesheets/rails_pulse/components/row.css +3 -17
- data/app/controllers/concerns/chart_table_concern.rb +28 -69
- data/app/controllers/concerns/deployment_markers_concern.rb +3 -1
- data/app/controllers/concerns/metric_card_concern.rb +6 -7
- data/app/controllers/rails_pulse/api/v1/base_controller.rb +133 -0
- data/app/controllers/rails_pulse/api/v1/capabilities_controller.rb +27 -0
- data/app/controllers/rails_pulse/api/v1/coverage_controller.rb +158 -0
- data/app/controllers/rails_pulse/api/v1/deployments_controller.rb +40 -0
- data/app/controllers/rails_pulse/api/v1/exceptions_controller.rb +77 -0
- data/app/controllers/rails_pulse/api/v1/insights_controller.rb +62 -0
- data/app/controllers/rails_pulse/api/v1/job_runs_controller.rb +34 -0
- data/app/controllers/rails_pulse/api/v1/jobs_controller.rb +172 -0
- data/app/controllers/rails_pulse/api/v1/queries_controller.rb +123 -0
- data/app/controllers/rails_pulse/api/v1/requests_controller.rb +74 -0
- data/app/controllers/rails_pulse/api/v1/routes_controller.rb +103 -0
- data/app/controllers/rails_pulse/application_controller.rb +1 -1
- data/app/controllers/rails_pulse/dashboard_controller.rb +25 -21
- data/app/controllers/rails_pulse/deployments_controller.rb +23 -13
- data/app/controllers/rails_pulse/exceptions_controller.rb +10 -12
- data/app/controllers/rails_pulse/jobs_controller.rb +1 -2
- data/app/controllers/rails_pulse/queries_controller.rb +6 -5
- data/app/controllers/rails_pulse/requests_controller.rb +15 -14
- data/app/controllers/rails_pulse/routes_controller.rb +4 -8
- data/app/helpers/rails_pulse/application_helper.rb +15 -0
- data/app/helpers/rails_pulse/chart_helper.rb +9 -2
- data/app/helpers/rails_pulse/formatting_helper.rb +11 -12
- data/app/javascript/rails_pulse/controllers/chart_controller.js +80 -43
- data/app/jobs/rails_pulse/backfill_summaries_job.rb +4 -2
- data/app/jobs/rails_pulse/cloud_health_job.rb +11 -0
- data/app/jobs/rails_pulse/cloud_sync_job.rb +11 -0
- data/app/jobs/rails_pulse/summary_job.rb +13 -0
- data/app/models/concerns/rails_pulse/has_metadata.rb +15 -0
- data/app/models/concerns/rails_pulse/taggable.rb +5 -12
- data/app/models/rails_pulse/cards/base.rb +20 -11
- data/app/models/rails_pulse/charts/base.rb +5 -18
- data/app/models/rails_pulse/charts/percentile_chart_base.rb +3 -5
- data/app/models/rails_pulse/cleanup_run.rb +32 -0
- data/app/models/rails_pulse/cloud/buffered_batch.rb +55 -0
- data/app/models/rails_pulse/cloud/installation.rb +55 -0
- data/app/models/rails_pulse/dashboard/charts/response_time_percentiles.rb +4 -5
- data/app/models/rails_pulse/dashboard/charts/throughput_and_errors.rb +4 -5
- data/app/models/rails_pulse/dashboard/concerns/attention_classification.rb +104 -0
- data/app/models/rails_pulse/dashboard/concerns/time_range_helper.rb +25 -2
- data/app/models/rails_pulse/dashboard/health_summary.rb +38 -13
- data/app/models/rails_pulse/dashboard/needs_attention.rb +31 -98
- data/app/models/rails_pulse/dashboard/storage_pressure.rb +39 -63
- data/app/models/rails_pulse/dashboard/storage_status.rb +130 -39
- data/app/models/rails_pulse/deployment.rb +2 -7
- data/app/models/rails_pulse/event.rb +41 -0
- data/app/models/rails_pulse/exception_occurrence.rb +1 -1
- data/app/models/rails_pulse/exceptions/cards/exception_rate.rb +3 -6
- data/app/models/rails_pulse/exceptions/cards/open_groups.rb +1 -4
- data/app/models/rails_pulse/exceptions/cards/total_occurrences.rb +3 -6
- data/app/models/rails_pulse/exceptions/charts/occurrence_volume.rb +4 -7
- data/app/models/rails_pulse/job_run.rb +1 -1
- data/app/models/rails_pulse/jobs/cards/failure_rate.rb +8 -14
- data/app/models/rails_pulse/jobs/cards/p95_duration.rb +8 -14
- data/app/models/rails_pulse/jobs/cards/total_runs.rb +5 -11
- data/app/models/rails_pulse/jobs/charts/duration.rb +1 -2
- data/app/models/rails_pulse/jobs/charts/execution_volume.rb +1 -1
- data/app/models/rails_pulse/jobs/charts/failure_rate.rb +1 -2
- data/app/models/rails_pulse/like_pattern.rb +35 -0
- data/app/models/rails_pulse/queries/cards/average_query_times.rb +6 -12
- data/app/models/rails_pulse/queries/cards/database_load.rb +0 -9
- data/app/models/rails_pulse/queries/cards/execution_rate.rb +5 -11
- data/app/models/rails_pulse/queries/cards/percentile_query_times.rb +6 -12
- data/app/models/rails_pulse/queries/charts/database_load.rb +5 -8
- data/app/models/rails_pulse/queries/charts/execution_volume.rb +1 -1
- data/app/models/rails_pulse/request.rb +1 -1
- data/app/models/rails_pulse/routes/cards/error_rates.rb +5 -15
- data/app/models/rails_pulse/routes/cards/percentile_response_times.rb +5 -17
- data/app/models/rails_pulse/routes/cards/request_count_totals.rb +5 -17
- data/app/models/rails_pulse/routes/charts/error_rate.rb +1 -1
- data/app/models/rails_pulse/routes/charts/request_volume.rb +1 -1
- data/app/models/rails_pulse/summary.rb +5 -0
- data/app/models/rails_pulse/tables/base.rb +5 -6
- data/app/models/rails_pulse/time_range.rb +309 -0
- data/app/models/rails_pulse/time_window.rb +15 -0
- data/app/models/rails_pulse/writer_heartbeat.rb +90 -0
- data/app/serializers/rails_pulse/api/v1/deployment_serializer.rb +23 -0
- data/app/serializers/rails_pulse/api/v1/exception_group_serializer.rb +23 -0
- data/app/serializers/rails_pulse/api/v1/exception_occurrence_serializer.rb +22 -0
- data/app/serializers/rails_pulse/api/v1/job_run_serializer.rb +28 -0
- data/app/serializers/rails_pulse/api/v1/job_serializer.rb +26 -0
- data/app/serializers/rails_pulse/api/v1/query_serializer.rb +25 -0
- data/app/serializers/rails_pulse/api/v1/request_serializer.rb +22 -0
- data/app/serializers/rails_pulse/api/v1/route_serializer.rb +19 -0
- data/app/services/rails_pulse/cloud/batch.rb +83 -0
- data/app/services/rails_pulse/cloud/client.rb +68 -0
- data/app/services/rails_pulse/cloud/combined_metrics.rb +49 -0
- data/app/services/rails_pulse/cloud/health_item.rb +77 -0
- data/app/services/rails_pulse/cloud/path_prefix.rb +50 -0
- data/app/services/rails_pulse/cloud/preview.rb +83 -0
- data/app/services/rails_pulse/cloud/query_shape.rb +195 -0
- data/app/services/rails_pulse/cloud/record_items.rb +60 -0
- data/app/services/rails_pulse/cloud/route_pattern.rb +111 -0
- data/app/services/rails_pulse/cloud/summary_items.rb +188 -0
- data/app/services/rails_pulse/cloud/sync.rb +275 -0
- data/app/services/rails_pulse/config_recommendations.rb +132 -0
- data/app/services/rails_pulse/deployment_comparison.rb +177 -0
- data/app/services/rails_pulse/exception_capture_service.rb +1 -1
- data/app/services/rails_pulse/operations/change_point.rb +1 -1
- data/app/services/rails_pulse/operations/comparison.rb +1 -1
- data/app/services/rails_pulse/period_insights.rb +131 -0
- data/app/services/rails_pulse/sql_query_normalizer.rb +222 -122
- data/app/services/rails_pulse/summary_service/from_child_periods.rb +161 -0
- data/app/services/rails_pulse/summary_service/from_raw_rows.rb +115 -0
- data/app/services/rails_pulse/summary_service/metrics.rb +38 -0
- data/app/services/rails_pulse/summary_service.rb +121 -133
- data/app/services/rails_pulse/tag_filter_service.rb +5 -5
- data/app/views/layouts/rails_pulse/_time_range_selector.html.erb +4 -1
- data/app/views/rails_pulse/components/_operation_details_popover.html.erb +1 -1
- data/app/views/rails_pulse/components/_table_head.html.erb +2 -2
- data/app/views/rails_pulse/dashboard/_chart_panel.html.erb +6 -1
- data/app/views/rails_pulse/dashboard/_deployments_panel.html.erb +2 -2
- data/app/views/rails_pulse/dashboard/_health_badge.html.erb +29 -20
- data/app/views/rails_pulse/dashboard/_storage_panel.html.erb +13 -3
- data/app/views/rails_pulse/dashboard/index.html.erb +16 -2
- data/app/views/rails_pulse/jobs/_chart_tabs.html.erb +1 -0
- data/app/views/rails_pulse/jobs/index.html.erb +1 -1
- data/app/views/rails_pulse/queries/_chart_tabs.html.erb +1 -0
- data/app/views/rails_pulse/queries/index.html.erb +1 -1
- data/app/views/rails_pulse/queries/show.html.erb +1 -1
- data/app/views/rails_pulse/requests/index.html.erb +1 -1
- data/app/views/rails_pulse/routes/_chart_tabs.html.erb +1 -0
- data/app/views/rails_pulse/routes/index.html.erb +1 -1
- data/app/views/rails_pulse/routes/show.html.erb +1 -1
- data/app/views/rails_pulse/shared/_aggregation_zone_badge.html.erb +4 -0
- data/app/views/rails_pulse/storage/show.html.erb +39 -2
- data/config/routes.rb +23 -0
- data/db/rails_pulse_cloud_schema.rb +58 -0
- data/db/rails_pulse_migrate/20260506000001_create_rails_pulse_exceptions.rb +1 -1
- data/db/rails_pulse_migrate/20260921000001_create_rails_pulse_events.rb +21 -0
- data/db/rails_pulse_schema.rb +20 -2
- data/exe/rails-pulse +7 -0
- data/lib/generators/rails_pulse/base_methods.rb +19 -0
- data/lib/generators/rails_pulse/install_cloud_generator.rb +84 -0
- data/lib/generators/rails_pulse/templates/db/rails_pulse_cloud_schema.rb +58 -0
- data/lib/generators/rails_pulse/templates/db/rails_pulse_schema.rb +20 -2
- data/lib/generators/rails_pulse/templates/migrations/install_rails_pulse_cloud_tables.rb +30 -0
- data/lib/generators/rails_pulse/templates/rails_pulse.rb +71 -19
- data/lib/generators/rails_pulse/upgrade_generator.rb +9 -16
- data/lib/rails_pulse/cleanup_service.rb +37 -20
- data/lib/rails_pulse/cli/agent_files/agents.md +89 -0
- data/lib/rails_pulse/cli/agent_files/claude_skill.md +182 -0
- data/lib/rails_pulse/cli/base_command.rb +37 -0
- data/lib/rails_pulse/cli/client.rb +64 -0
- data/lib/rails_pulse/cli/config.rb +87 -0
- data/lib/rails_pulse/cli/configure.rb +75 -0
- data/lib/rails_pulse/cli/coverage.rb +91 -0
- data/lib/rails_pulse/cli/deployments.rb +54 -0
- data/lib/rails_pulse/cli/exceptions.rb +116 -0
- data/lib/rails_pulse/cli/formatter.rb +31 -0
- data/lib/rails_pulse/cli/insights.rb +70 -0
- data/lib/rails_pulse/cli/install.rb +145 -0
- data/lib/rails_pulse/cli/job_runs.rb +53 -0
- data/lib/rails_pulse/cli/jobs.rb +71 -0
- data/lib/rails_pulse/cli/main.rb +76 -0
- data/lib/rails_pulse/cli/mcp.rb +47 -0
- data/lib/rails_pulse/cli/queries.rb +67 -0
- data/lib/rails_pulse/cli/requests.rb +57 -0
- data/lib/rails_pulse/cli/routes.rb +73 -0
- data/lib/rails_pulse/configuration.rb +100 -2
- data/lib/rails_pulse/current.rb +11 -0
- data/lib/rails_pulse/engine.rb +12 -3
- data/lib/rails_pulse/installers/config_updater.rb +9 -2
- data/lib/rails_pulse/job_run_collector.rb +15 -15
- data/lib/rails_pulse/mcp/server.rb +78 -0
- data/lib/rails_pulse/mcp/tools/coverage.rb +96 -0
- data/lib/rails_pulse/mcp/tools/deployments.rb +117 -0
- data/lib/rails_pulse/mcp/tools/endpoint.rb +167 -0
- data/lib/rails_pulse/mcp/tools/errors.rb +104 -0
- data/lib/rails_pulse/mcp/tools/exception_detail.rb +118 -0
- data/lib/rails_pulse/mcp/tools/exceptions.rb +128 -0
- data/lib/rails_pulse/mcp/tools/helpers.rb +162 -0
- data/lib/rails_pulse/mcp/tools/insights.rb +99 -0
- data/lib/rails_pulse/mcp/tools/jobs.rb +155 -0
- data/lib/rails_pulse/mcp/tools/queries.rb +142 -0
- data/lib/rails_pulse/mcp/tools/routes.rb +108 -0
- data/lib/rails_pulse/mcp/tools/slow_requests.rb +118 -0
- data/lib/rails_pulse/middleware/request_collector.rb +9 -9
- data/lib/rails_pulse/scoped_inflector.rb +35 -0
- data/lib/rails_pulse/statistics.rb +27 -0
- data/lib/rails_pulse/subscribers/exception_subscriber.rb +2 -2
- data/lib/rails_pulse/subscribers/operation_subscriber.rb +15 -29
- data/lib/rails_pulse/tasks/status_reporter.rb +201 -0
- data/lib/rails_pulse/tracker.rb +76 -14
- data/lib/rails_pulse/version.rb +1 -1
- data/lib/rails_pulse.rb +0 -4
- data/lib/tasks/rails_pulse.rake +6 -0
- data/lib/tasks/rails_pulse_tasks.rake +7 -0
- data/public/rails-pulse-assets/rails-pulse.css +1 -1
- data/public/rails-pulse-assets/rails-pulse.js +13 -13
- metadata +123 -24
- data/app/controllers/concerns/page_timings.rb +0 -10
- data/app/controllers/concerns/response_range_concern.rb +0 -45
- data/app/controllers/concerns/time_range_concern.rb +0 -152
- data/app/controllers/concerns/zoom_range_concern.rb +0 -86
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
require "net/http"
|
|
2
|
+
require "openssl"
|
|
3
|
+
require "uri"
|
|
4
|
+
require "json"
|
|
5
|
+
require_relative "config"
|
|
6
|
+
|
|
7
|
+
module RailsPulse
|
|
8
|
+
module CLI
|
|
9
|
+
# HTTP client for the read-only JSON API under <mount>/api/v1.
|
|
10
|
+
class Client
|
|
11
|
+
class ApiError < StandardError; end
|
|
12
|
+
|
|
13
|
+
def initialize(config = nil)
|
|
14
|
+
@config = config || Config.load
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def get(path, params = {})
|
|
18
|
+
uri = URI("#{@config.url}#{@config.mount_path}/api/v1#{path}")
|
|
19
|
+
uri.query = URI.encode_www_form(params) unless params.empty?
|
|
20
|
+
|
|
21
|
+
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
|
|
22
|
+
req = Net::HTTP::Get.new(uri)
|
|
23
|
+
req["X-Rails-Pulse-Token"] = @config.token
|
|
24
|
+
response = http.request(req)
|
|
25
|
+
raise_for(response) unless response.is_a?(Net::HTTPSuccess)
|
|
26
|
+
parse(response, uri)
|
|
27
|
+
end
|
|
28
|
+
rescue URI::InvalidURIError => e
|
|
29
|
+
raise ApiError, "the configured URL and mount path do not form a valid URL (#{e.message})"
|
|
30
|
+
rescue OpenSSL::SSL::SSLError => e
|
|
31
|
+
raise ApiError, "TLS failed talking to #{uri.host}:#{uri.port} (#{e.message})"
|
|
32
|
+
rescue SocketError, SystemCallError, Net::OpenTimeout, Net::ReadTimeout => e
|
|
33
|
+
raise ApiError, "could not connect to #{uri.host}:#{uri.port} (#{e.message})"
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
# A 200 that is not JSON is almost always the wrong URL or mount path
|
|
39
|
+
# answered by the host app itself: a login page or a catch-all route.
|
|
40
|
+
def parse(response, uri)
|
|
41
|
+
JSON.parse(response.body.to_s)
|
|
42
|
+
rescue JSON::ParserError
|
|
43
|
+
type = response["Content-Type"].to_s.split(";").first
|
|
44
|
+
raise ApiError, "#{uri} answered with #{type.to_s.empty? ? 'a non-JSON body' : type} rather than JSON. " \
|
|
45
|
+
"Check the URL and mount path ('rails-pulse configure' or RAILS_PULSE_MOUNT_PATH)"
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def raise_for(response)
|
|
49
|
+
body = begin
|
|
50
|
+
JSON.parse(response.body.to_s)
|
|
51
|
+
rescue JSON::ParserError
|
|
52
|
+
{}
|
|
53
|
+
end
|
|
54
|
+
body = {} unless body.is_a?(Hash)
|
|
55
|
+
|
|
56
|
+
# The API explains a 400 or 401 in the body ("Invalid sort. Valid
|
|
57
|
+
# values: ..."); relay it rather than only the status line.
|
|
58
|
+
message = "#{response.code} #{response.message}".strip
|
|
59
|
+
message = "#{message}: #{body["error"]}" if body["error"].is_a?(String) && !body["error"].empty?
|
|
60
|
+
raise ApiError, message
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
require "yaml"
|
|
2
|
+
|
|
3
|
+
module RailsPulse
|
|
4
|
+
module CLI
|
|
5
|
+
# Where the CLI and MCP server find the app to talk to. Environment
|
|
6
|
+
# variables win; otherwise ~/.rails-pulse (or RAILS_PULSE_CONFIG), written
|
|
7
|
+
# by `rails-pulse configure`.
|
|
8
|
+
class Config
|
|
9
|
+
class ConfigError < StandardError; end
|
|
10
|
+
|
|
11
|
+
DEFAULT_PATH = "~/.rails-pulse".freeze
|
|
12
|
+
DEFAULT_MOUNT_PATH = "/rails_pulse".freeze
|
|
13
|
+
URL_SCHEMES = %r{\Ahttps?://\S+\z}i
|
|
14
|
+
|
|
15
|
+
# The file holds the API token, so nobody else on the machine may read it.
|
|
16
|
+
FILE_MODE = 0o600
|
|
17
|
+
|
|
18
|
+
attr_reader :url, :token, :mount_path
|
|
19
|
+
|
|
20
|
+
def initialize(url:, token:, mount_path: DEFAULT_MOUNT_PATH)
|
|
21
|
+
@url = url.to_s.chomp("/")
|
|
22
|
+
@token = token.to_s
|
|
23
|
+
# A root mount is an empty prefix, not "/", or every path gains "//".
|
|
24
|
+
trimmed = mount_path.to_s.delete_prefix("/").chomp("/")
|
|
25
|
+
@mount_path = trimmed.empty? ? "" : "/#{trimmed}"
|
|
26
|
+
|
|
27
|
+
raise ConfigError, "RAILS_PULSE_URL must start with http:// or https:// (got #{@url.inspect})" unless @url.match?(URL_SCHEMES)
|
|
28
|
+
# The API path is appended to the URL, so anything after the host
|
|
29
|
+
# belongs in the mount path instead.
|
|
30
|
+
if @url.match?(/[?#]/)
|
|
31
|
+
raise ConfigError, "RAILS_PULSE_URL must not contain a query string or fragment (got #{@url.inspect})"
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def self.path
|
|
36
|
+
File.expand_path(ENV.fetch("RAILS_PULSE_CONFIG", DEFAULT_PATH))
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def self.load
|
|
40
|
+
url = ENV["RAILS_PULSE_URL"]
|
|
41
|
+
token = ENV["RAILS_PULSE_TOKEN"]
|
|
42
|
+
mount_path = ENV["RAILS_PULSE_MOUNT_PATH"]
|
|
43
|
+
|
|
44
|
+
# Complete credentials in the environment never touch the file, so a
|
|
45
|
+
# broken ~/.rails-pulse cannot block a setup that does not use it.
|
|
46
|
+
if url.nil? || token.nil?
|
|
47
|
+
data = read_file
|
|
48
|
+
url ||= data["url"]
|
|
49
|
+
token ||= data["token"]
|
|
50
|
+
mount_path ||= data["mount_path"]
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
url = url.to_s.chomp("/")
|
|
54
|
+
|
|
55
|
+
raise ConfigError, "RAILS_PULSE_URL is not set. Run 'rails-pulse configure'" if url.empty?
|
|
56
|
+
raise ConfigError, "RAILS_PULSE_TOKEN is not set. Run 'rails-pulse configure'" if token.to_s.empty?
|
|
57
|
+
|
|
58
|
+
new(url: url, token: token, mount_path: mount_path || DEFAULT_MOUNT_PATH)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def self.write!(url:, token:, mount_path: nil)
|
|
62
|
+
data = { "url" => url.to_s.chomp("/"), "token" => token.to_s }
|
|
63
|
+
data["mount_path"] = mount_path if mount_path && mount_path != DEFAULT_MOUNT_PATH
|
|
64
|
+
# perm: applies only when the file is created, so an existing file is
|
|
65
|
+
# narrowed before the token is written into it.
|
|
66
|
+
File.chmod(FILE_MODE, path) if File.exist?(path)
|
|
67
|
+
File.write(path, YAML.dump(data), perm: FILE_MODE)
|
|
68
|
+
File.chmod(FILE_MODE, path)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# The parsed config file, or an empty hash when there is none yet.
|
|
72
|
+
def self.read_file
|
|
73
|
+
data = YAML.safe_load_file(path)
|
|
74
|
+
raise ConfigError, "#{path} should be a YAML mapping with url and token keys" unless data.nil? || data.is_a?(Hash)
|
|
75
|
+
|
|
76
|
+
data || {}
|
|
77
|
+
rescue Errno::ENOENT
|
|
78
|
+
{}
|
|
79
|
+
rescue Errno::EACCES, Errno::EISDIR => e
|
|
80
|
+
raise ConfigError, "#{path} cannot be read (#{e.message}). Fix its permissions or run 'rails-pulse configure'"
|
|
81
|
+
rescue Psych::SyntaxError => e
|
|
82
|
+
raise ConfigError, "#{path} is not valid YAML (#{e.message}). Fix it or run 'rails-pulse configure'"
|
|
83
|
+
end
|
|
84
|
+
private_class_method :read_file
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
require_relative "base_command"
|
|
2
|
+
require_relative "config"
|
|
3
|
+
require_relative "client"
|
|
4
|
+
|
|
5
|
+
module RailsPulse
|
|
6
|
+
module CLI
|
|
7
|
+
class Configure < BaseCommand
|
|
8
|
+
default_task :setup
|
|
9
|
+
|
|
10
|
+
desc "setup", "Prompt for URL and API token, test the connection, and write ~/.rails-pulse"
|
|
11
|
+
long_desc <<~DESC
|
|
12
|
+
Interactively configure credentials for the Rails Pulse JSON API.
|
|
13
|
+
|
|
14
|
+
Prompts for:
|
|
15
|
+
- Rails Pulse URL (e.g. http://localhost:3000)
|
|
16
|
+
- API token (matches config.api_token in the app's Rails Pulse initializer)
|
|
17
|
+
- Mount path (where the engine is mounted; default /rails_pulse)
|
|
18
|
+
|
|
19
|
+
Existing values are shown as defaults — press Enter to keep them.
|
|
20
|
+
The token is not echoed as you type it.
|
|
21
|
+
|
|
22
|
+
A test request is made before saving. If it fails, no file is written.
|
|
23
|
+
Credentials are saved to ~/.rails-pulse (never to the project directory),
|
|
24
|
+
readable only by you.
|
|
25
|
+
|
|
26
|
+
You can also skip this command and set environment variables instead:
|
|
27
|
+
RAILS_PULSE_URL, RAILS_PULSE_TOKEN, RAILS_PULSE_MOUNT_PATH (optional)
|
|
28
|
+
DESC
|
|
29
|
+
def setup
|
|
30
|
+
current = begin
|
|
31
|
+
Config.load
|
|
32
|
+
rescue Config::ConfigError
|
|
33
|
+
nil
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
url = ask_with_default("Rails Pulse URL", current&.url)
|
|
37
|
+
token = ask_with_default("API token", current&.token, echo: false, mask: true)
|
|
38
|
+
mount_path = ask_with_default("Mount path", current&.mount_path || Config::DEFAULT_MOUNT_PATH)
|
|
39
|
+
|
|
40
|
+
say "Testing connection..."
|
|
41
|
+
begin
|
|
42
|
+
test_config = Config.new(url: url, token: token, mount_path: mount_path)
|
|
43
|
+
Client.new(test_config).get("/routes", { limit: 1 })
|
|
44
|
+
rescue Config::ConfigError, Client::ApiError => e
|
|
45
|
+
say "Connection failed: #{e.message}", :red
|
|
46
|
+
exit 1
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
Config.write!(url: url, token: token, mount_path: test_config.mount_path)
|
|
50
|
+
say "Configuration saved to #{Config.path}", :green
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
private
|
|
54
|
+
|
|
55
|
+
# Prompts with the current value as the default; an empty answer keeps
|
|
56
|
+
# it. A masked default shows only that a value exists, never the token.
|
|
57
|
+
def ask_with_default(label, default, echo: true, mask: false)
|
|
58
|
+
prompt = if default.to_s.empty?
|
|
59
|
+
"#{label}:"
|
|
60
|
+
else
|
|
61
|
+
"#{label} [#{mask ? 'keep current' : default}]:"
|
|
62
|
+
end
|
|
63
|
+
# Hiding input needs a terminal; piped input has nothing to hide from.
|
|
64
|
+
hide = !echo && terminal?
|
|
65
|
+
answer = ask(prompt, echo: !hide)
|
|
66
|
+
say "" if hide
|
|
67
|
+
answer.to_s.empty? ? default.to_s : answer
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def terminal?
|
|
71
|
+
$stdin.tty?
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
require_relative "base_command"
|
|
2
|
+
require_relative "formatter"
|
|
3
|
+
|
|
4
|
+
module RailsPulse
|
|
5
|
+
module CLI
|
|
6
|
+
class Coverage < BaseCommand
|
|
7
|
+
desc "show", "Report what has been recorded, how recently, and any gaps"
|
|
8
|
+
long_desc <<~DESC
|
|
9
|
+
Answers whether the data is there, rather than a question about the data: which
|
|
10
|
+
application and environment answered, the span of requests, job runs and exceptions
|
|
11
|
+
held, how far summaries have been generated, what retention keeps, and whether the
|
|
12
|
+
background writer has dropped anything.
|
|
13
|
+
|
|
14
|
+
Read this before concluding that nothing went wrong. An empty error list means
|
|
15
|
+
"nothing recorded", which is only the same as "nothing happened" when collection was
|
|
16
|
+
healthy over the window in question.
|
|
17
|
+
|
|
18
|
+
Use --json for the full structure.
|
|
19
|
+
DESC
|
|
20
|
+
option :json, type: :boolean, default: false, desc: "Output raw JSON"
|
|
21
|
+
def show
|
|
22
|
+
with_error_handling do
|
|
23
|
+
data = client.get("/coverage", {})
|
|
24
|
+
capabilities = client.get("/capabilities", {})
|
|
25
|
+
next say(JSON.pretty_generate(data.merge("capabilities" => capabilities))) if options[:json]
|
|
26
|
+
|
|
27
|
+
print_installation(capabilities)
|
|
28
|
+
print_telemetry(data["telemetry"] || {})
|
|
29
|
+
print_summaries(data["summaries"] || {})
|
|
30
|
+
print_retention(data["retention"] || {})
|
|
31
|
+
print_collection(data["collection"] || {})
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
def print_installation(capabilities)
|
|
38
|
+
say "Installation"
|
|
39
|
+
say " application #{capabilities['application'] || '—'} (#{capabilities['environment']})"
|
|
40
|
+
say " version #{capabilities['rails_pulse_version']}"
|
|
41
|
+
say ""
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def print_telemetry(telemetry)
|
|
45
|
+
say "Telemetry"
|
|
46
|
+
telemetry.each do |kind, info|
|
|
47
|
+
if info["tracked"] == false
|
|
48
|
+
say " #{kind.ljust(12)} not tracked — #{info['reason']}"
|
|
49
|
+
elsif info["newest"]
|
|
50
|
+
say " #{kind.ljust(12)} #{info['count']} rows, #{info['oldest']} → #{info['newest']}"
|
|
51
|
+
else
|
|
52
|
+
say " #{kind.ljust(12)} nothing recorded"
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def print_summaries(summaries)
|
|
58
|
+
say ""
|
|
59
|
+
say "Summaries"
|
|
60
|
+
say " hourly #{summaries['hourly_from'] || '—'} → #{summaries['hourly_through'] || '—'}#{summaries['stale'] ? ' (stale)' : ''}"
|
|
61
|
+
say " #{summaries['note']}" if summaries["note"]
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def print_retention(retention)
|
|
65
|
+
say ""
|
|
66
|
+
say "Retention"
|
|
67
|
+
say " raw records #{days(retention['raw_records'])}"
|
|
68
|
+
say " hourly #{days(retention['hourly_summaries'])}"
|
|
69
|
+
say " events #{days(retention['events'])}"
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def print_collection(collection)
|
|
73
|
+
say ""
|
|
74
|
+
say "Collection"
|
|
75
|
+
if collection["known"] == false
|
|
76
|
+
say " unknown — #{collection['reason']}"
|
|
77
|
+
return
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
say " writers #{collection['live_writers']} live, queue #{collection['queue_depth']}/#{collection['queue_size']}"
|
|
81
|
+
say " dropped #{collection['dropped_last_hour']} in the last hour"
|
|
82
|
+
say " last beat #{collection['last_heartbeat_at'] || 'never'}"
|
|
83
|
+
say " #{collection['note']}" if collection["note"]
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def days(value)
|
|
87
|
+
value ? "#{value['days']} days" : "—"
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
require_relative "base_command"
|
|
2
|
+
require_relative "formatter"
|
|
3
|
+
|
|
4
|
+
module RailsPulse
|
|
5
|
+
module CLI
|
|
6
|
+
class Deployments < BaseCommand
|
|
7
|
+
COLUMNS = [
|
|
8
|
+
[ "Revision", 14, :short_revision ],
|
|
9
|
+
[ "Started", 25, :started_at ],
|
|
10
|
+
[ "Finished", 25, :finished_at ],
|
|
11
|
+
[ "Compared", 17, :compared ]
|
|
12
|
+
].freeze
|
|
13
|
+
|
|
14
|
+
desc "list", "List recorded deployments"
|
|
15
|
+
long_desc <<~DESC
|
|
16
|
+
Returns deployments ordered by most recent first. A deployment still in progress has
|
|
17
|
+
no finish time.
|
|
18
|
+
|
|
19
|
+
Each deployment is compared across the hours it ran in: the hour before it started
|
|
20
|
+
against the hour after it finished. degraded when average or p95 response time is
|
|
21
|
+
more than 1.5x and at least 50ms worse afterwards, or the error rate more than 1.25x
|
|
22
|
+
and at least a percentage point worse; insufficient_data when either hour had fewer
|
|
23
|
+
than 10 requests; pending until the hour after has been summarized, or unavailable
|
|
24
|
+
when it never will be because SummaryJob is not running.
|
|
25
|
+
|
|
26
|
+
Filter by time window (ISO 8601):
|
|
27
|
+
--since 2026-06-01T00:00:00Z
|
|
28
|
+
--until 2026-06-01T23:59:59Z
|
|
29
|
+
|
|
30
|
+
Find a deployment by its full revision or the start of one:
|
|
31
|
+
--revision a1b2c3d
|
|
32
|
+
|
|
33
|
+
Use --json to get the full revision, duration, metadata and per-metric comparison.
|
|
34
|
+
DESC
|
|
35
|
+
option :limit, type: :numeric, default: 25, desc: "Max records to return (1–500)"
|
|
36
|
+
option :offset, type: :numeric, default: 0, desc: "Number of records to skip (for pagination)"
|
|
37
|
+
option :since, type: :string, desc: "Return deployments started at or after this time (ISO 8601)"
|
|
38
|
+
option :until, type: :string, desc: "Return deployments started at or before this time (ISO 8601)"
|
|
39
|
+
option :revision, type: :string, desc: "Full revision or the start of one"
|
|
40
|
+
option :json, type: :boolean, default: false, desc: "Output raw JSON including meta envelope"
|
|
41
|
+
def list
|
|
42
|
+
with_error_handling do
|
|
43
|
+
params = { limit: options[:limit], offset: options[:offset] }
|
|
44
|
+
params[:since] = options[:since] if options[:since]
|
|
45
|
+
params[:until] = options[:until] if options[:until]
|
|
46
|
+
params[:revision] = options[:revision] if options[:revision]
|
|
47
|
+
result = client.get("/deployments", params)
|
|
48
|
+
(result["data"] || []).each { |row| row["compared"] = row.dig("comparison", "outcome") } unless options[:json]
|
|
49
|
+
Formatter.render(result, json: options[:json], columns: COLUMNS)
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
require_relative "base_command"
|
|
2
|
+
require_relative "formatter"
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module RailsPulse
|
|
6
|
+
module CLI
|
|
7
|
+
class Exceptions < BaseCommand
|
|
8
|
+
COLUMNS = [
|
|
9
|
+
[ "ID", 6, :id ],
|
|
10
|
+
[ "Class", 36, :exception_class ],
|
|
11
|
+
[ "Location", 40, :location ],
|
|
12
|
+
[ "Status", 9, :status ],
|
|
13
|
+
[ "Count", 6, :occurrence_count ],
|
|
14
|
+
[ "Last seen", 25, :last_seen_at ]
|
|
15
|
+
].freeze
|
|
16
|
+
|
|
17
|
+
desc "list", "List exception groups"
|
|
18
|
+
long_desc <<~DESC
|
|
19
|
+
Returns exception groups — one row per exception class and location — ordered by most
|
|
20
|
+
recently seen first. Each row carries the group's lifecycle status (open, resolved,
|
|
21
|
+
ignored), how many times it has occurred, and when it was first and last seen.
|
|
22
|
+
|
|
23
|
+
Filter by status:
|
|
24
|
+
--status open
|
|
25
|
+
|
|
26
|
+
Search the class name or location:
|
|
27
|
+
--search RecordNotFound
|
|
28
|
+
|
|
29
|
+
Look up one group by its fingerprint, as listed with --json or by Rails Pulse Cloud:
|
|
30
|
+
--fingerprint 9b2c4e7a1f3d5c8b
|
|
31
|
+
|
|
32
|
+
Filter by when the group was last seen (ISO 8601):
|
|
33
|
+
--since 2026-06-01T00:00:00Z
|
|
34
|
+
--until 2026-06-01T23:59:59Z
|
|
35
|
+
|
|
36
|
+
Sort by last_seen_at (default), first_seen_at, or occurrence_count:
|
|
37
|
+
--sort occurrence_count
|
|
38
|
+
|
|
39
|
+
Use --json to get the latest message, fingerprint, and the full response envelope.
|
|
40
|
+
DESC
|
|
41
|
+
option :limit, type: :numeric, default: 25, desc: "Max records to return (1–500)"
|
|
42
|
+
option :offset, type: :numeric, default: 0, desc: "Number of records to skip (for pagination)"
|
|
43
|
+
option :since, type: :string, desc: "Return groups last seen at or after this time (ISO 8601)"
|
|
44
|
+
option :until, type: :string, desc: "Return groups last seen at or before this time (ISO 8601)"
|
|
45
|
+
option :status, type: :string, desc: "Filter by status (open, resolved, ignored)"
|
|
46
|
+
option :search, type: :string, desc: "Substring match on exception class or location"
|
|
47
|
+
option :fingerprint, type: :string, desc: "Exact fingerprint of one group"
|
|
48
|
+
option :sort, type: :string, desc: "Sort by last_seen_at, first_seen_at, or occurrence_count"
|
|
49
|
+
option :json, type: :boolean, default: false, desc: "Output raw JSON including meta envelope"
|
|
50
|
+
def list
|
|
51
|
+
with_error_handling do
|
|
52
|
+
params = { limit: options[:limit], offset: options[:offset] }
|
|
53
|
+
params[:since] = options[:since] if options[:since]
|
|
54
|
+
params[:until] = options[:until] if options[:until]
|
|
55
|
+
params[:status] = options[:status] if options[:status]
|
|
56
|
+
params[:search] = options[:search] if options[:search]
|
|
57
|
+
params[:fingerprint] = options[:fingerprint] if options[:fingerprint]
|
|
58
|
+
params[:sort] = options[:sort] if options[:sort]
|
|
59
|
+
result = client.get("/exceptions", params)
|
|
60
|
+
Formatter.render(result, json: options[:json], columns: COLUMNS)
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
desc "show ID", "Show one exception group with its recent occurrences and backtraces"
|
|
65
|
+
long_desc <<~DESC
|
|
66
|
+
Prints the group's class, location, status and counts, then each of its most recent
|
|
67
|
+
occurrences with request, environment, deploy SHA, filtered params, and the backtrace.
|
|
68
|
+
|
|
69
|
+
rails-pulse exceptions show 42
|
|
70
|
+
rails-pulse exceptions show 42 --occurrences 10 --json
|
|
71
|
+
DESC
|
|
72
|
+
option :occurrences, type: :numeric, default: 3, desc: "Most recent occurrences to include (1–20)"
|
|
73
|
+
option :json, type: :boolean, default: false, desc: "Output raw JSON"
|
|
74
|
+
def show(id)
|
|
75
|
+
with_error_handling do
|
|
76
|
+
result = client.get("/exceptions/#{id.to_i}", { occurrences: options[:occurrences] })
|
|
77
|
+
if options[:json]
|
|
78
|
+
puts JSON.pretty_generate(result)
|
|
79
|
+
else
|
|
80
|
+
render_group(result["data"] || {})
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
no_commands do
|
|
86
|
+
def render_group(group)
|
|
87
|
+
say ""
|
|
88
|
+
say "#{group["exception_class"]} ##{group["id"]}", :bold
|
|
89
|
+
say "═" * 60
|
|
90
|
+
say " Location: #{group["location"] || "—"}"
|
|
91
|
+
say " Status: #{group["status"]}"
|
|
92
|
+
say " Count: #{group["occurrence_count"]}"
|
|
93
|
+
say " First seen: #{group["first_seen_at"]}"
|
|
94
|
+
say " Last seen: #{group["last_seen_at"]}"
|
|
95
|
+
say " Message: #{group["message"]}" if group["message"]
|
|
96
|
+
|
|
97
|
+
Array(group["occurrences"]).each_with_index do |occurrence, index|
|
|
98
|
+
say ""
|
|
99
|
+
say "Occurrence #{index + 1} — #{occurrence["occurred_at"]}", :bold
|
|
100
|
+
say " Request: #{occurrence["request_method"]} #{occurrence["request_url"]}" if occurrence["request_url"]
|
|
101
|
+
say " Env: #{occurrence["environment"]}" if occurrence["environment"]
|
|
102
|
+
say " Deploy: #{occurrence["deploy_sha"]}" if occurrence["deploy_sha"]
|
|
103
|
+
say " Params: #{occurrence["request_params"].to_json}" if occurrence["request_params"]
|
|
104
|
+
say " Message: #{occurrence["message"]}" if occurrence["message"] && occurrence["message"] != group["message"]
|
|
105
|
+
Array(occurrence["backtrace"]).each do |frame|
|
|
106
|
+
location = frame.is_a?(Hash) ? "#{frame["file"]}:#{frame["line"]}#{frame["method"] ? " in #{frame["method"]}" : ""}" : frame.to_s
|
|
107
|
+
app = location.start_with?("app/", "lib/", "config/")
|
|
108
|
+
say " #{app ? "▸" : " "} #{location}", app ? :green : nil
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
say ""
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
|
|
3
|
+
module RailsPulse
|
|
4
|
+
module CLI
|
|
5
|
+
class Formatter
|
|
6
|
+
def self.render(response, json: false, columns: [])
|
|
7
|
+
if json
|
|
8
|
+
puts JSON.pretty_generate(response)
|
|
9
|
+
else
|
|
10
|
+
render_table(response["data"], columns)
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def self.render_table(rows, columns)
|
|
15
|
+
return puts "(no results)" if rows.empty?
|
|
16
|
+
|
|
17
|
+
header = columns.map { |header, width, _key| header.upcase.ljust(width) }.join(" ")
|
|
18
|
+
puts header
|
|
19
|
+
puts "-" * header.length
|
|
20
|
+
|
|
21
|
+
rows.each do |row|
|
|
22
|
+
line = columns.map do |_header, width, key|
|
|
23
|
+
val = row[key.to_s].to_s
|
|
24
|
+
val.length > width ? "#{val[0, width - 1]}…" : val.ljust(width)
|
|
25
|
+
end.join(" ")
|
|
26
|
+
puts line
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
require_relative "base_command"
|
|
2
|
+
require_relative "formatter"
|
|
3
|
+
|
|
4
|
+
module RailsPulse
|
|
5
|
+
module CLI
|
|
6
|
+
class Insights < BaseCommand
|
|
7
|
+
PERIODS = %w[hour day week month].freeze
|
|
8
|
+
|
|
9
|
+
desc "show", "What needs attention over one period, and whether the thresholds fit it"
|
|
10
|
+
long_desc <<~DESC
|
|
11
|
+
Reads one summary period (an hour, day, week or month) and lists the routes, queries
|
|
12
|
+
and jobs past their slow or critical thresholds, critical first, ten at most. Then
|
|
13
|
+
checks route_thresholds and query_thresholds against the period's slowest routes and
|
|
14
|
+
most expensive queries, and suggests an initializer line when a slow threshold is
|
|
15
|
+
exceeded by most of them or a critical threshold is never approached.
|
|
16
|
+
|
|
17
|
+
Defaults to the last complete week. --at picks the period containing that time:
|
|
18
|
+
--period day --at 2026-06-10T00:00:00Z
|
|
19
|
+
|
|
20
|
+
Use --json for the full structure, including record ids and current thresholds.
|
|
21
|
+
DESC
|
|
22
|
+
option :period, type: :string, default: "week", enum: PERIODS, desc: "Summary period to read"
|
|
23
|
+
option :at, type: :string, desc: "A time inside the period (ISO 8601); default the last complete one"
|
|
24
|
+
option :json, type: :boolean, default: false, desc: "Output raw JSON"
|
|
25
|
+
def show
|
|
26
|
+
with_error_handling do
|
|
27
|
+
params = { period: options[:period] }
|
|
28
|
+
params[:at] = options[:at] if options[:at]
|
|
29
|
+
data = client.get("/insights", params)
|
|
30
|
+
next say(JSON.pretty_generate(data)) if options[:json]
|
|
31
|
+
|
|
32
|
+
print_period(data["period"] || {})
|
|
33
|
+
print_attention(data["needs_attention"] || {})
|
|
34
|
+
print_recommendations(data["threshold_recommendations"] || [])
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
def print_period(period)
|
|
41
|
+
say "#{period['type'].to_s.capitalize} from #{period['start']} to #{period['end']}"
|
|
42
|
+
say " not fully summarized yet — what follows may be incomplete" unless period["summarized"]
|
|
43
|
+
say ""
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def print_attention(attention)
|
|
47
|
+
say "Needs attention"
|
|
48
|
+
items = Array(attention["critical"]) + Array(attention["warning"])
|
|
49
|
+
return say(" nothing past its thresholds") if items.empty?
|
|
50
|
+
|
|
51
|
+
items.each do |item|
|
|
52
|
+
say " #{item['severity'].to_s.upcase.ljust(8)} #{item['type'].to_s.ljust(5)} #{item['name']}"
|
|
53
|
+
say " #{item['reason']}"
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def print_recommendations(recommendations)
|
|
58
|
+
say ""
|
|
59
|
+
say "Threshold recommendations"
|
|
60
|
+
return say(" none — the thresholds fit this period") if recommendations.empty?
|
|
61
|
+
|
|
62
|
+
recommendations.each do |rec|
|
|
63
|
+
say " #{rec['title']}"
|
|
64
|
+
say " #{rec['detail']}"
|
|
65
|
+
say " #{rec['config_snippet']}"
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|