zharp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +33 -0
- data/LICENSE +21 -0
- data/README.md +170 -0
- data/lib/generators/zharp/install/install_generator.rb +39 -0
- data/lib/generators/zharp/install/templates/zharp.rb.tt +15 -0
- data/lib/zharp/bodies.rb +386 -0
- data/lib/zharp/capture.rb +361 -0
- data/lib/zharp/config.rb +129 -0
- data/lib/zharp/context.rb +133 -0
- data/lib/zharp/db_errors.rb +229 -0
- data/lib/zharp/errors.rb +158 -0
- data/lib/zharp/fork.rb +70 -0
- data/lib/zharp/headers.rb +63 -0
- data/lib/zharp/instrumentations.rb +17 -0
- data/lib/zharp/logs.rb +128 -0
- data/lib/zharp/rack.rb +148 -0
- data/lib/zharp/railtie.rb +47 -0
- data/lib/zharp/setup.rb +206 -0
- data/lib/zharp/sidekiq.rb +13 -0
- data/lib/zharp/source.rb +118 -0
- data/lib/zharp/stamp.rb +124 -0
- data/lib/zharp/version.rb +5 -0
- data/lib/zharp.rb +218 -0
- metadata +341 -0
data/lib/zharp/rack.rb
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Zharp
|
|
4
|
+
# Any Rack app: Sinatra, Hanami, Roda, Grape, plain Rack. In config.ru:
|
|
5
|
+
#
|
|
6
|
+
# require "zharp"
|
|
7
|
+
# Zharp.init(service: "shop") # optional; ZHARP_KEY etc. from the environment otherwise
|
|
8
|
+
# use Zharp::Rack
|
|
9
|
+
# run App
|
|
10
|
+
#
|
|
11
|
+
# One server span per request (OpenTelemetry's Rack instrumentation, continuing the caller's
|
|
12
|
+
# traceparent), named by route, with every request and response header (secrets as [REDACTED]),
|
|
13
|
+
# and an exception that escapes reported once. Rails needs none of this: its Railtie adds it.
|
|
14
|
+
class Rack
|
|
15
|
+
# Built with the app (config.ru), so a preloading server (Puma's preload_app!, Unicorn's
|
|
16
|
+
# preload_app) starts Zharp once, before it forks.
|
|
17
|
+
def initialize(app)
|
|
18
|
+
@app = app
|
|
19
|
+
@inner = Headers.new(app)
|
|
20
|
+
@traced = build
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def call(env)
|
|
24
|
+
@traced.call(env)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
# OpenTelemetry's Rack middleware around Zharp's.
|
|
30
|
+
def build
|
|
31
|
+
Zharp.init unless Zharp.active?
|
|
32
|
+
instrumentation = ::OpenTelemetry::Instrumentation::Rack::Instrumentation.instance
|
|
33
|
+
instrumentation.install({}) unless instrumentation.installed?
|
|
34
|
+
klass, *args = instrumentation.middleware_args
|
|
35
|
+
klass.new(@inner, *args)
|
|
36
|
+
rescue StandardError, NameError => e
|
|
37
|
+
Zharp.debug("rack: #{e.message}")
|
|
38
|
+
@inner
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Zharp's layer inside the server span: headers, the exception that escaped, the route's name.
|
|
42
|
+
# Rails inserts it right after OpenTelemetry's middleware.
|
|
43
|
+
class Headers
|
|
44
|
+
METHOD_ONLY = /\A[A-Z]+\z/
|
|
45
|
+
|
|
46
|
+
def initialize(app)
|
|
47
|
+
@app = app
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Where a framework leaves an exception it turned into an error page itself (Hanami, Roda and
|
|
51
|
+
# other Rack apps use rack.exception).
|
|
52
|
+
HANDLED_ERRORS = %w[rack.exception sinatra.error action_dispatch.exception].freeze
|
|
53
|
+
|
|
54
|
+
def call(env)
|
|
55
|
+
# The path as it arrived: a mounted app or router may rewrite PATH_INFO on the way in.
|
|
56
|
+
path = env["PATH_INFO"].to_s
|
|
57
|
+
span = Rack.server_span
|
|
58
|
+
secret = Zharp.options&.capture_headers
|
|
59
|
+
if span.recording? && secret
|
|
60
|
+
span.add_attributes(Zharp::Headers.attributes("request", Zharp::Headers.from_env(env), secret))
|
|
61
|
+
end
|
|
62
|
+
bodies = Zharp.options&.capture_bodies
|
|
63
|
+
tee = Capture.start_request(env, bodies) if bodies && span.recording?
|
|
64
|
+
status, headers, body = @app.call(env)
|
|
65
|
+
if span.recording?
|
|
66
|
+
span.add_attributes(Zharp::Headers.attributes("response", headers, secret)) if secret && headers
|
|
67
|
+
report_rendered_error(span, env, status)
|
|
68
|
+
Rack.name_span(span, env, path)
|
|
69
|
+
body = Capture.finish_request(span, env, tee, status, headers, body, bodies) if tee
|
|
70
|
+
end
|
|
71
|
+
[status, headers, body]
|
|
72
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
73
|
+
if span&.recording? && !Errors.reported?(e) && !e.is_a?(SystemExit) && !e.is_a?(SignalException)
|
|
74
|
+
span.record_exception(e, attributes: { "zharp.handled" => "false" })
|
|
75
|
+
span.status = ::OpenTelemetry::Trace::Status.error(e.class.name)
|
|
76
|
+
# No response left the app: the server (Puma, Unicorn) answers 500.
|
|
77
|
+
span.set_attribute("http.response.status_code", 500)
|
|
78
|
+
Capture.add(span) { Capture.request_attributes(env, tee, bodies) } if tee
|
|
79
|
+
Rack.name_span(span, env, path)
|
|
80
|
+
end
|
|
81
|
+
raise
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
private
|
|
85
|
+
|
|
86
|
+
# A 5xx the framework rendered from an exception it caught: that exception escaped the app's
|
|
87
|
+
# code, so it is reported, once, as unhandled.
|
|
88
|
+
def report_rendered_error(span, env, status)
|
|
89
|
+
return unless status.to_i >= 500
|
|
90
|
+
|
|
91
|
+
error = HANDLED_ERRORS.lazy.map { |key| env[key] }.find { |e| e.is_a?(Exception) }
|
|
92
|
+
return if error.nil? || Errors.reported?(error)
|
|
93
|
+
|
|
94
|
+
span.record_exception(error, attributes: { "zharp.handled" => "false" })
|
|
95
|
+
span.status = ::OpenTelemetry::Trace::Status.error(error.class.name)
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
class << self
|
|
100
|
+
def server_span
|
|
101
|
+
if defined?(::OpenTelemetry::Instrumentation::Rack) && ::OpenTelemetry::Instrumentation::Rack.respond_to?(:current_span)
|
|
102
|
+
span = ::OpenTelemetry::Instrumentation::Rack.current_span
|
|
103
|
+
return span if span.recording?
|
|
104
|
+
end
|
|
105
|
+
::OpenTelemetry::Trace.current_span
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Sinatra, Grape and Rails name the span by route themselves. Otherwise "GET /orders/:id": the
|
|
109
|
+
# path with its ids replaced, so one route is one name.
|
|
110
|
+
def name_span(span, env, path = nil)
|
|
111
|
+
return unless span.respond_to?(:name) && span.name.to_s.match?(Headers::METHOD_ONLY)
|
|
112
|
+
|
|
113
|
+
route = env["zharp.route"] || env["sinatra.route"]&.split&.last || quantize(path || env["PATH_INFO"].to_s)
|
|
114
|
+
span.name = "#{env['REQUEST_METHOD']} #{route}"
|
|
115
|
+
span.set_attribute("http.route", route) unless span.attributes&.key?("http.route")
|
|
116
|
+
rescue StandardError
|
|
117
|
+
nil
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
ID_SEGMENT = /\A(?:\d+|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|[0-9a-f]{16,}|[A-Za-z0-9_-]{22,})\z/i
|
|
121
|
+
|
|
122
|
+
def quantize(path)
|
|
123
|
+
return "/" if path.empty?
|
|
124
|
+
|
|
125
|
+
path.split("/", -1).map { |segment| segment.match?(ID_SEGMENT) ? ":id" : segment }.join("/")
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# A Rack app inside another (Sinatra's own middleware inside `use Zharp::Rack`, or a Rails engine)
|
|
130
|
+
# gets no second server span: the outer one is the request.
|
|
131
|
+
module NestedSpan
|
|
132
|
+
def on_start(request, response)
|
|
133
|
+
outer = request.env["otel.rack.context_info"]
|
|
134
|
+
return if outer.is_a?(Hash) && !outer.empty?
|
|
135
|
+
|
|
136
|
+
super
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def self.patch_nested
|
|
141
|
+
%w[Stable Dup Old].each do |name|
|
|
142
|
+
mod = ::OpenTelemetry::Instrumentation::Rack::Middlewares.const_get(name) rescue next
|
|
143
|
+
handler = mod.const_get(:EventHandler) rescue next
|
|
144
|
+
handler.prepend(NestedSpan) unless handler.ancestors.include?(NestedSpan)
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
end
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Zharp
|
|
4
|
+
# Rails starts Zharp by itself: Bundler.require loads the gem, and this Railtie calls Zharp.init
|
|
5
|
+
# once config/initializers have run (config/initializers/zharp.rb may call it first, with options).
|
|
6
|
+
# It adds Zharp's header layer next to OpenTelemetry's Rack middleware and subscribes to
|
|
7
|
+
# Rails.error, which Rails reports every unhandled request and job error to.
|
|
8
|
+
class Railtie < ::Rails::Railtie
|
|
9
|
+
initializer "zharp.start", after: :load_config_initializers do |app|
|
|
10
|
+
Zharp.init unless Zharp.active?
|
|
11
|
+
if Zharp.active? && Zharp.options.tracing
|
|
12
|
+
# OpenTelemetry's middleware was inserted at 0 when it was installed: ours goes right after it.
|
|
13
|
+
app.middleware.insert_after(0, Zharp::Rack::Headers)
|
|
14
|
+
end
|
|
15
|
+
Rails.error.subscribe(Zharp::ErrorSubscriber.new) if Zharp.active? && Rails.respond_to?(:error)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# semantic_logger (rails_semantic_logger) may be set up after Zharp started.
|
|
19
|
+
config.after_initialize do
|
|
20
|
+
Setup.install_semantic_logger if Zharp.active?
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
generators do
|
|
24
|
+
require_relative "../generators/zharp/install/install_generator"
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Rails.error: handled reports (Rails.error.handle, Rails.error.report) and unhandled ones (from
|
|
29
|
+
# the executor around requests and jobs). On the request's span when there is one, else a log
|
|
30
|
+
# record; never twice for one exception.
|
|
31
|
+
class ErrorSubscriber
|
|
32
|
+
def report(error, handled:, severity: :error, context: {}, source: nil)
|
|
33
|
+
return if Errors.reported?(error) || severity == :info
|
|
34
|
+
|
|
35
|
+
span = Rack.server_span
|
|
36
|
+
if span.recording? && !handled
|
|
37
|
+
span.record_exception(error, attributes: { "zharp.handled" => "false" })
|
|
38
|
+
else
|
|
39
|
+
attributes = {}
|
|
40
|
+
attributes["zharp.source"] = source.to_s if source
|
|
41
|
+
Zharp.report(error, handled: handled, attributes: attributes)
|
|
42
|
+
end
|
|
43
|
+
rescue StandardError => e
|
|
44
|
+
Zharp.debug("Rails.error: #{e.message}")
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
data/lib/zharp/setup.rb
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "logger"
|
|
4
|
+
require "socket"
|
|
5
|
+
|
|
6
|
+
module Zharp
|
|
7
|
+
# Wiring the official OpenTelemetry SDK to Zharp: exporters, resource, propagators, processors and
|
|
8
|
+
# the instrumentations. No tracer of our own.
|
|
9
|
+
module Setup
|
|
10
|
+
State = Struct.new(:options, :tracer_provider, :logger_provider, :tracer, :logger, :instrumented, keyword_init: true)
|
|
11
|
+
|
|
12
|
+
# Every instrumentation in opentelemetry-instrumentation-all switches on when its library is
|
|
13
|
+
# loaded. Jobs run in the trace of the request that queued them (OpenTelemetry's default is a
|
|
14
|
+
# linked trace of their own).
|
|
15
|
+
INSTRUMENTATION = {
|
|
16
|
+
"OpenTelemetry::Instrumentation::Sidekiq" => { propagation_style: :child },
|
|
17
|
+
"OpenTelemetry::Instrumentation::ActiveJob" => { propagation_style: :child }
|
|
18
|
+
}.freeze
|
|
19
|
+
|
|
20
|
+
# Root spans: the share sample_rate keeps, except a database or Redis call with nothing around
|
|
21
|
+
# it: job runners' pollers (Solid Queue, GoodJob, Delayed Job, Sidekiq's heartbeat) and Action
|
|
22
|
+
# Cable's pings would be a trace every second that says nothing. The same calls inside a
|
|
23
|
+
# request, a job or a span of your own (Zharp.start_span) are traced as usual.
|
|
24
|
+
class Sampler
|
|
25
|
+
DROP = ::OpenTelemetry::SDK::Trace::Samplers::Result.new(decision: ::OpenTelemetry::SDK::Trace::Samplers::Decision::DROP, tracestate: ::OpenTelemetry::Trace::Tracestate::DEFAULT)
|
|
26
|
+
|
|
27
|
+
# Active Record spans on the job runners' own tables (SolidQueue::Process query, …).
|
|
28
|
+
JOB_RUNNER_MODELS = /\A(SolidQueue|GoodJob|Delayed)::/
|
|
29
|
+
|
|
30
|
+
def initialize(root)
|
|
31
|
+
@root = root
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def should_sample?(trace_id:, parent_context:, links:, name:, kind:, attributes:)
|
|
35
|
+
return DROP if kind == :client && attributes && (attributes["db.system"] || attributes["db.system.name"])
|
|
36
|
+
return DROP if name == "ActiveRecord.transaction" || JOB_RUNNER_MODELS.match?(name.to_s)
|
|
37
|
+
|
|
38
|
+
@root.should_sample?(trace_id: trace_id, parent_context: parent_context, links: links, name: name, kind: kind, attributes: attributes)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def description = "ZharpSampler{#{@root.description}}"
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
module_function
|
|
45
|
+
|
|
46
|
+
def start(options, exporters: nil, batch: true)
|
|
47
|
+
quiet_opentelemetry
|
|
48
|
+
resource = resource_for(options)
|
|
49
|
+
span_exporter, log_exporter = exporters || default_exporters(options)
|
|
50
|
+
|
|
51
|
+
tracer_provider = nil
|
|
52
|
+
if options.tracing
|
|
53
|
+
tracer_provider = ::OpenTelemetry::SDK::Trace::TracerProvider.new(
|
|
54
|
+
resource: resource,
|
|
55
|
+
sampler: ::OpenTelemetry::SDK::Trace::Samplers.parent_based(
|
|
56
|
+
root: Sampler.new(::OpenTelemetry::SDK::Trace::Samplers.trace_id_ratio_based(options.sample_rate))
|
|
57
|
+
)
|
|
58
|
+
)
|
|
59
|
+
tracer_provider.add_span_processor(Context::SpanProcessor.new)
|
|
60
|
+
tracer_provider.add_span_processor(Capture::UrlRedaction.new)
|
|
61
|
+
# Before the exporter, so a held exception is reported before its trace ships.
|
|
62
|
+
Errors.db_errors = DbErrors::Processor.new(method(:send_held_error), queries: options.capture_query_errors)
|
|
63
|
+
tracer_provider.add_span_processor(Errors.db_errors)
|
|
64
|
+
tracer_provider.add_span_processor(batch ? ::OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(span_exporter) : ::OpenTelemetry::SDK::Trace::Export::SimpleSpanProcessor.new(span_exporter))
|
|
65
|
+
::OpenTelemetry.tracer_provider = tracer_provider
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
logger_provider = ::OpenTelemetry::SDK::Logs::LoggerProvider.new(resource: resource)
|
|
69
|
+
logger_provider.add_log_record_processor(Context::LogProcessor.new)
|
|
70
|
+
logger_provider.add_log_record_processor(batch ? ::OpenTelemetry::SDK::Logs::Export::BatchLogRecordProcessor.new(log_exporter) : ::OpenTelemetry::SDK::Logs::Export::SimpleLogRecordProcessor.new(log_exporter))
|
|
71
|
+
::OpenTelemetry.logger_provider = logger_provider
|
|
72
|
+
|
|
73
|
+
::OpenTelemetry.propagation = ::OpenTelemetry::Context::Propagation::CompositeTextMapPropagator.compose_propagators(
|
|
74
|
+
[::OpenTelemetry::Trace::Propagation::TraceContext.text_map_propagator, ::OpenTelemetry::Baggage::Propagation.text_map_propagator]
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
Fork.register(span_exporter)
|
|
78
|
+
Fork.register(log_exporter)
|
|
79
|
+
Fork.install
|
|
80
|
+
|
|
81
|
+
Source.enabled = options.capture_source_context
|
|
82
|
+
Errors.patch_record_exception if options.tracing
|
|
83
|
+
state = State.new(
|
|
84
|
+
options: options,
|
|
85
|
+
tracer_provider: tracer_provider,
|
|
86
|
+
logger_provider: logger_provider,
|
|
87
|
+
tracer: (tracer_provider || ::OpenTelemetry::Trace::TracerProvider.new).tracer("zharp", VERSION),
|
|
88
|
+
logger: logger_provider.logger(name: "zharp", version: VERSION),
|
|
89
|
+
instrumented: []
|
|
90
|
+
)
|
|
91
|
+
# Before OpenTelemetry's Net::HTTP patch, so it runs inside that span.
|
|
92
|
+
Capture.install_net_http if options.tracing && options.capture_bodies
|
|
93
|
+
state.instrumented = instrument(options) if options.tracing && options.instrument
|
|
94
|
+
start_logs(options)
|
|
95
|
+
state
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def resource_for(options)
|
|
99
|
+
attributes = {
|
|
100
|
+
"service.name" => options.service,
|
|
101
|
+
"telemetry.sdk.name" => "zharp",
|
|
102
|
+
"telemetry.sdk.language" => "ruby",
|
|
103
|
+
"telemetry.sdk.version" => VERSION,
|
|
104
|
+
"process.pid" => Process.pid,
|
|
105
|
+
"process.runtime.name" => RUBY_ENGINE,
|
|
106
|
+
"process.runtime.version" => RUBY_VERSION
|
|
107
|
+
}
|
|
108
|
+
attributes["service.version"] = options.release if options.release
|
|
109
|
+
attributes["deployment.environment.name"] = options.environment if options.environment
|
|
110
|
+
attributes["host.name"] = Socket.gethostname rescue nil
|
|
111
|
+
# Resource.create adds OTEL_RESOURCE_ATTRIBUTES; ours win.
|
|
112
|
+
::OpenTelemetry::SDK::Resources::Resource.create(attributes.compact)
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def default_exporters(options)
|
|
116
|
+
headers = { "Authorization" => "Bearer #{options.key}" }
|
|
117
|
+
[
|
|
118
|
+
::OpenTelemetry::Exporter::OTLP::Exporter.new(endpoint: "#{options.endpoint}/v1/traces", headers: headers, compression: "gzip"),
|
|
119
|
+
::OpenTelemetry::Exporter::OTLP::Logs::LogsExporter.new(endpoint: "#{options.endpoint}/v1/logs", headers: headers, compression: "gzip")
|
|
120
|
+
]
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def instrument(options)
|
|
124
|
+
config = INSTRUMENTATION.merge(options.instrumentation.transform_keys(&:to_s)) { |_k, ours, theirs| ours.merge(theirs) }
|
|
125
|
+
::OpenTelemetry::Instrumentation.registry.install_all(config)
|
|
126
|
+
Rack.patch_nested if defined?(::OpenTelemetry::Instrumentation::Rack::Middlewares)
|
|
127
|
+
install_sidekiq
|
|
128
|
+
installed_names
|
|
129
|
+
rescue StandardError => e
|
|
130
|
+
Config.announce("Could not instrument: #{e.message}")
|
|
131
|
+
[]
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def installed_names
|
|
135
|
+
list = ::OpenTelemetry::Instrumentation.registry.instance_variable_get(:@instrumentation) || []
|
|
136
|
+
list.map(&:instance).select(&:installed?).map { |i| i.name.split("::").last.gsub(/([a-z\d])([A-Z])/, '\1_\2').downcase }
|
|
137
|
+
rescue StandardError
|
|
138
|
+
[]
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# Sidekiq's server middleware records a job's exception on its span; the error handler reports
|
|
142
|
+
# what fails outside it. Both mark the exception, so it is counted once.
|
|
143
|
+
def install_sidekiq
|
|
144
|
+
return unless defined?(::Sidekiq) && ::Sidekiq.respond_to?(:configure_server)
|
|
145
|
+
|
|
146
|
+
::Sidekiq.configure_server do |config|
|
|
147
|
+
handler = ->(error, _context = nil, _config = nil) { Zharp.report(error, handled: false) unless Errors.reported?(error) }
|
|
148
|
+
config.error_handlers << handler if config.respond_to?(:error_handlers)
|
|
149
|
+
end
|
|
150
|
+
rescue StandardError => e
|
|
151
|
+
Zharp.debug("sidekiq: #{e.message}")
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def start_logs(options)
|
|
155
|
+
Stamp.enabled = options.log_correlation
|
|
156
|
+
Stamp.install_logger
|
|
157
|
+
Logs.sending = options.logs
|
|
158
|
+
Logs.install_logger
|
|
159
|
+
install_semantic_logger
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def install_semantic_logger
|
|
163
|
+
Stamp.install_semantic_logger
|
|
164
|
+
Logs.install_semantic_logger
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# An exception held on a span inside a request or job that nothing reported: the app caught
|
|
168
|
+
# it. A failed query carries its query and the app's call site as its stack.
|
|
169
|
+
def send_held_error(error, stack, attributes, span_context)
|
|
170
|
+
return if Errors.reported?(error)
|
|
171
|
+
|
|
172
|
+
Errors.mark(error, handled: true)
|
|
173
|
+
attrs = Errors.attributes(error, true).merge(attributes)
|
|
174
|
+
attrs["exception.stacktrace"] = stack if stack
|
|
175
|
+
attrs["zharp.mechanism"] = "db" if attributes.key?("db.system")
|
|
176
|
+
context = ::OpenTelemetry::Trace.context_with_span(::OpenTelemetry::Trace.non_recording_span(span_context))
|
|
177
|
+
Logs.emit("ERROR", 17, "#{attrs['exception.type']}: #{attrs['exception.message']}", attrs, context)
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# OpenTelemetry's own messages: warnings only, and each one once (a wrong key would otherwise
|
|
181
|
+
# print a line every few seconds).
|
|
182
|
+
def quiet_opentelemetry
|
|
183
|
+
logger = ::Logger.new($stderr, level: ::Logger::WARN, progname: "Zharp/OpenTelemetry")
|
|
184
|
+
logger.instance_variable_set(:@__zharp_own, true)
|
|
185
|
+
::OpenTelemetry.logger = logger
|
|
186
|
+
seen = {}
|
|
187
|
+
lock = Mutex.new
|
|
188
|
+
::OpenTelemetry.error_handler = lambda do |exception: nil, message: nil|
|
|
189
|
+
text = [message, exception&.message].compact.join(" - ")
|
|
190
|
+
first = lock.synchronize do
|
|
191
|
+
seen.clear if seen.size > 100
|
|
192
|
+
seen[text[0, 200]] ? false : (seen[text[0, 200]] = true)
|
|
193
|
+
end
|
|
194
|
+
logger.warn("OpenTelemetry error: #{text}") if first
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
def describe(state)
|
|
199
|
+
options = state.options
|
|
200
|
+
env = options.environment ? %(environment "#{options.environment}") : "no environment set"
|
|
201
|
+
what = state.instrumented.empty? ? "no supported libraries found" : state.instrumented.join(", ")
|
|
202
|
+
mode = options.tracing ? "Application Monitoring active" : "Application Monitoring active, logs only"
|
|
203
|
+
%(#{mode} — service "#{options.service}", #{env} (#{what}))
|
|
204
|
+
end
|
|
205
|
+
end
|
|
206
|
+
end
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# A Sidekiq process outside Rails. At the top of the file Sidekiq loads (sidekiq -r ./worker.rb):
|
|
4
|
+
#
|
|
5
|
+
# require "zharp/sidekiq"
|
|
6
|
+
#
|
|
7
|
+
# Sidekiq keeps only one -r, so this goes in that file rather than on the command line. ZHARP_KEY
|
|
8
|
+
# and the other settings come from the environment. Rails apps need nothing: the Railtie starts
|
|
9
|
+
# Zharp in Sidekiq too.
|
|
10
|
+
require "sidekiq"
|
|
11
|
+
require "zharp"
|
|
12
|
+
|
|
13
|
+
Zharp.init
|
data/lib/zharp/source.rb
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "rbconfig"
|
|
5
|
+
|
|
6
|
+
module Zharp
|
|
7
|
+
# The code around an error's lines, read on the app's own server and sent with the error as
|
|
8
|
+
# zharp.source_context. Ruby runs its source files, so the lines are right there: Zharp shows
|
|
9
|
+
# exactly the code that ran, with no repository link and nothing to upload. Only the app's own
|
|
10
|
+
# frames (never gems, Ruby's library, OpenTelemetry or Zharp), the innermost five, five lines
|
|
11
|
+
# either side, each line capped. capture_source_context: false (or
|
|
12
|
+
# ZHARP_CAPTURE_SOURCE_CONTEXT=false) turns it off.
|
|
13
|
+
module Source
|
|
14
|
+
ATTRIBUTE = "zharp.source_context"
|
|
15
|
+
AROUND = 5
|
|
16
|
+
MAX_FRAMES = 5
|
|
17
|
+
MAX_LINE = 300
|
|
18
|
+
MAX_BYTES = 12_000
|
|
19
|
+
MAX_FILES = 200
|
|
20
|
+
|
|
21
|
+
@enabled = true
|
|
22
|
+
@cache = {}
|
|
23
|
+
@lock = Mutex.new
|
|
24
|
+
|
|
25
|
+
class << self
|
|
26
|
+
attr_accessor :enabled
|
|
27
|
+
|
|
28
|
+
# The JSON for an error's frames, innermost first: [{file, line, pre, code, post}].
|
|
29
|
+
# frames: [[path as the stack prints it, path to read, line]].
|
|
30
|
+
def context_of(frames)
|
|
31
|
+
return nil unless enabled
|
|
32
|
+
|
|
33
|
+
out = []
|
|
34
|
+
seen = {}
|
|
35
|
+
frames.each do |shown, readable, line|
|
|
36
|
+
break if out.size >= MAX_FRAMES
|
|
37
|
+
next if line.nil? || line < 1 || seen[[shown, line]] || !Paths.app?(readable)
|
|
38
|
+
|
|
39
|
+
seen[[shown, line]] = true
|
|
40
|
+
lines = lines_of(readable)
|
|
41
|
+
next if lines.nil? || line > lines.size
|
|
42
|
+
|
|
43
|
+
pre = lines[[0, line - 1 - AROUND].max...(line - 1)].map { |l| cut(l) }
|
|
44
|
+
post = (lines[line, AROUND] || []).map { |l| cut(l) }
|
|
45
|
+
out << { file: shown, line: line, pre: pre, code: cut(lines[line - 1]), post: post }
|
|
46
|
+
end
|
|
47
|
+
until out.empty?
|
|
48
|
+
text = JSON.generate(out)
|
|
49
|
+
return text if text.bytesize <= MAX_BYTES
|
|
50
|
+
|
|
51
|
+
out.pop
|
|
52
|
+
end
|
|
53
|
+
nil
|
|
54
|
+
rescue StandardError
|
|
55
|
+
nil
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def of_exception(error)
|
|
59
|
+
context_of(frames_of(error))
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# An exception's frames, innermost first.
|
|
63
|
+
def frames_of(error)
|
|
64
|
+
locations = error.backtrace_locations
|
|
65
|
+
if locations
|
|
66
|
+
locations.map { |l| [l.path, l.absolute_path || l.path, l.lineno] }
|
|
67
|
+
else
|
|
68
|
+
(error.backtrace || []).filter_map do |text|
|
|
69
|
+
m = /\A(.+?):(\d+)(?::in |\z)/.match(text)
|
|
70
|
+
[m[1], m[1], m[2].to_i] if m
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def cut(text)
|
|
76
|
+
text = text.to_s.chomp
|
|
77
|
+
text.length <= MAX_LINE ? text : "#{text[0, MAX_LINE - 1]}…"
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def lines_of(path)
|
|
81
|
+
@lock.synchronize do
|
|
82
|
+
return @cache[path] if @cache.key?(path)
|
|
83
|
+
|
|
84
|
+
@cache.shift if @cache.size >= MAX_FILES
|
|
85
|
+
@cache[path] =
|
|
86
|
+
begin
|
|
87
|
+
File.file?(path) && File.size(path) < 2_000_000 ? File.readlines(path, chomp: true, encoding: "UTF-8") : nil
|
|
88
|
+
rescue StandardError
|
|
89
|
+
nil
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Where a frame's file lives: the app's own code, or not (gems, Ruby's library, the VM, Zharp).
|
|
97
|
+
module Paths
|
|
98
|
+
ZHARP_DIR = File.expand_path("..", __dir__)
|
|
99
|
+
|
|
100
|
+
class << self
|
|
101
|
+
def app?(path)
|
|
102
|
+
return false if path.nil? || path.empty? || path.start_with?("<") || path.start_with?("(")
|
|
103
|
+
return false if library_roots.any? { |root| path.start_with?(root) }
|
|
104
|
+
|
|
105
|
+
!path.match?(%r{[\\/]gems[\\/]|[\\/]lib[\\/]ruby[\\/]\d})
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def library_roots
|
|
109
|
+
@library_roots ||= begin
|
|
110
|
+
roots = [ZHARP_DIR, RbConfig::CONFIG["rubylibprefix"], RbConfig::CONFIG["rubylibdir"]]
|
|
111
|
+
roots.concat(Gem.path) if defined?(Gem)
|
|
112
|
+
roots << ::Bundler.bundle_path.to_s if defined?(::Bundler) && (::Bundler.bundle_path rescue nil)
|
|
113
|
+
roots.compact.reject(&:empty?).map { |r| File.expand_path(r) + "/" }.uniq
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
data/lib/zharp/stamp.rb
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Zharp
|
|
6
|
+
# The request's ids at the end of each log line written in it: " trace_id=<32 hex> span_id=<16 hex>"
|
|
7
|
+
# on a text line, two fields on a JSON line (which stays valid JSON). The Zharp agent reads the
|
|
8
|
+
# app's output and links each line to its request by them. Lines outside a request, blank lines,
|
|
9
|
+
# lines already carrying a trace_id and Zharp's own lines are left as they are.
|
|
10
|
+
module Stamp
|
|
11
|
+
module_function
|
|
12
|
+
|
|
13
|
+
# The current span's ids as lowercase hex; nil outside a request.
|
|
14
|
+
def current_ids(context = nil)
|
|
15
|
+
span_context = ::OpenTelemetry::Trace.current_span(context).context
|
|
16
|
+
return nil unless span_context.valid?
|
|
17
|
+
|
|
18
|
+
[span_context.hex_trace_id, span_context.hex_span_id]
|
|
19
|
+
rescue StandardError
|
|
20
|
+
nil
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def json_object?(text)
|
|
24
|
+
JSON.parse(text).is_a?(Hash)
|
|
25
|
+
rescue JSON::ParserError, TypeError
|
|
26
|
+
false
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def stamp_line(line, ids)
|
|
30
|
+
trace_id, span_id = ids
|
|
31
|
+
newline = line[/\r?\n\z/] || ""
|
|
32
|
+
body = line[0, line.length - newline.length]
|
|
33
|
+
trimmed = body.rstrip
|
|
34
|
+
return line if trimmed.strip.empty? || body.include?("trace_id") || body.include?("[Zharp]")
|
|
35
|
+
|
|
36
|
+
tail = body[trimmed.length..]
|
|
37
|
+
start = trimmed.length - trimmed.lstrip.length
|
|
38
|
+
if trimmed.end_with?("}") && trimmed[start] == "{" && json_object?(trimmed[start..])
|
|
39
|
+
close = trimmed.rindex("}")
|
|
40
|
+
inner = trimmed[(start + 1)...close].strip
|
|
41
|
+
fields = %("trace_id":"#{trace_id}","span_id":"#{span_id}")
|
|
42
|
+
return "#{trimmed[0...close]}#{inner.empty? ? '' : ','}#{fields}}#{tail}#{newline}"
|
|
43
|
+
end
|
|
44
|
+
"#{trimmed} trace_id=#{trace_id} span_id=#{span_id}#{tail}#{newline}"
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# One record (which may span lines: a backtrace) stamped once, at the end of its last line.
|
|
48
|
+
def stamp_record(text, ids = nil)
|
|
49
|
+
return text unless text.is_a?(String) && !text.empty? && !text.include?("trace_id")
|
|
50
|
+
|
|
51
|
+
ids ||= current_ids
|
|
52
|
+
return text unless ids
|
|
53
|
+
|
|
54
|
+
lines = text.split(/(?<=\n)/)
|
|
55
|
+
index = lines.rindex { |l| !l.strip.empty? }
|
|
56
|
+
return text unless index
|
|
57
|
+
|
|
58
|
+
lines[index] = stamp_line(lines[index], ids)
|
|
59
|
+
lines.join
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# On Ruby's Logger (so Rails.logger, ActiveSupport::Logger, lograge): each formatted line.
|
|
63
|
+
module LoggerPatch
|
|
64
|
+
def format_message(severity, datetime, progname, msg)
|
|
65
|
+
text = super
|
|
66
|
+
Stamp.enabled ? Stamp.stamp_record(text) : text
|
|
67
|
+
rescue StandardError
|
|
68
|
+
text
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
class << self
|
|
73
|
+
attr_accessor :enabled
|
|
74
|
+
end
|
|
75
|
+
@enabled = false
|
|
76
|
+
|
|
77
|
+
def install_logger
|
|
78
|
+
::Logger.prepend(LoggerPatch) unless ::Logger.ancestors.include?(LoggerPatch)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# semantic_logger formats on its own thread: the ids are taken when the line is logged, and put
|
|
82
|
+
# on the line when its formatter writes it.
|
|
83
|
+
module SemanticLogPatch
|
|
84
|
+
def initialize(*args, **kwargs, &block)
|
|
85
|
+
super
|
|
86
|
+
@__zharp_ids = Stamp.current_ids
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
attr_reader :__zharp_ids
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
module SemanticFormatterPatch
|
|
93
|
+
def call(log, logger)
|
|
94
|
+
out = super
|
|
95
|
+
ids = log.respond_to?(:__zharp_ids) ? log.__zharp_ids : nil
|
|
96
|
+
return out unless Stamp.enabled && ids
|
|
97
|
+
|
|
98
|
+
case out
|
|
99
|
+
when String then Stamp.stamp_record(out, ids)
|
|
100
|
+
when Hash then out.merge(trace_id: ids[0], span_id: ids[1])
|
|
101
|
+
else out
|
|
102
|
+
end
|
|
103
|
+
rescue StandardError
|
|
104
|
+
out
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def install_semantic_logger
|
|
109
|
+
return false unless defined?(::SemanticLogger::Log) && defined?(::SemanticLogger::Formatters)
|
|
110
|
+
|
|
111
|
+
::SemanticLogger::Log.prepend(SemanticLogPatch) unless ::SemanticLogger::Log.ancestors.include?(SemanticLogPatch)
|
|
112
|
+
::SemanticLogger::Formatters.constants.each do |name|
|
|
113
|
+
klass = ::SemanticLogger::Formatters.const_get(name)
|
|
114
|
+
next unless klass.is_a?(Class) && klass.method_defined?(:call)
|
|
115
|
+
next if klass.ancestors.include?(SemanticFormatterPatch) || klass.instance_method(:call).owner != klass
|
|
116
|
+
|
|
117
|
+
klass.prepend(SemanticFormatterPatch)
|
|
118
|
+
rescue StandardError, LoadError
|
|
119
|
+
next
|
|
120
|
+
end
|
|
121
|
+
true
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|