chronos-ruby 0.2.0.pre.1 → 0.3.0.pre.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +25 -0
- data/README.md +35 -18
- data/docs/adr/ADR-005-sanitize-before-backlog.md +2 -2
- data/docs/adr/ADR-011-bounded-resilience-and-remote-control.md +27 -0
- data/docs/architecture.md +5 -4
- data/docs/compatibility.md +6 -6
- data/docs/configuration.md +17 -0
- data/docs/data-collected.md +2 -2
- data/docs/examples/plain-ruby.md +6 -0
- data/docs/modules/async-queue.md +6 -2
- data/docs/modules/notice-pipeline.md +2 -1
- data/docs/modules/remote-configuration.md +54 -0
- data/docs/modules/retry-backlog.md +60 -0
- data/docs/modules/transport.md +5 -3
- data/docs/performance.md +27 -2
- data/docs/privacy-lgpd.md +4 -2
- data/docs/troubleshooting.md +1 -1
- data/lib/chronos/adapters/net_http_transport.rb +21 -3
- data/lib/chronos/agent.rb +16 -8
- data/lib/chronos/application/capture_exception.rb +20 -14
- data/lib/chronos/application/circuit_breaker.rb +70 -0
- data/lib/chronos/application/delivery_pipeline.rb +248 -0
- data/lib/chronos/application/remote_configuration.rb +173 -0
- data/lib/chronos/application/retry_policy.rb +57 -0
- data/lib/chronos/configuration.rb +101 -19
- data/lib/chronos/core/payload_serializer.rb +4 -2
- data/lib/chronos/internal/bounded_queue.rb +1 -1
- data/lib/chronos/internal/memory_backlog.rb +65 -0
- data/lib/chronos/internal/worker_pool.rb +5 -6
- data/lib/chronos/ports/transport.rb +4 -3
- data/lib/chronos/version.rb +1 -1
- data/lib/chronos.rb +6 -1
- metadata +9 -1
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
require "net/http"
|
|
2
2
|
require "openssl"
|
|
3
|
+
require "json"
|
|
3
4
|
require "stringio"
|
|
4
5
|
require "uri"
|
|
5
6
|
require "zlib"
|
|
@@ -10,16 +11,17 @@ module Chronos
|
|
|
10
11
|
#
|
|
11
12
|
# @responsibility Send serialized events over bounded HTTPS requests.
|
|
12
13
|
# @motivation Use the Ruby standard library to preserve legacy compatibility.
|
|
13
|
-
# @limits It classifies failures but
|
|
14
|
+
# @limits It classifies failures but leaves retry and backlog policy to the application layer.
|
|
14
15
|
# @collaborators Configuration::Snapshot, SerializedEvent, and TransportResult.
|
|
15
16
|
# @thread_safety Creates a new Net::HTTP connection per call and synchronizes health state.
|
|
16
17
|
# @compatibility Ruby 2.2.10 through Ruby 2.6; no Rails dependency.
|
|
17
18
|
# @example
|
|
18
19
|
# result = transport.send_event(serialized_event)
|
|
19
20
|
# @errors Network, TLS, and HTTP errors are returned, never raised to callers.
|
|
20
|
-
# @performance One bounded network request per
|
|
21
|
+
# @performance One bounded network request per delivery attempt in version 0.3.
|
|
21
22
|
class NetHttpTransport
|
|
22
23
|
EVENT_PATH = "/api/v1/events".freeze
|
|
24
|
+
REMOTE_CONFIGURATION_HEADER = "X-Chronos-Remote-Configuration".freeze
|
|
23
25
|
|
|
24
26
|
def initialize(config, logger = nil)
|
|
25
27
|
@config = config
|
|
@@ -100,7 +102,11 @@ module Chronos
|
|
|
100
102
|
def classify(response)
|
|
101
103
|
code = response.code.to_i
|
|
102
104
|
options = {:status_code => code}
|
|
103
|
-
|
|
105
|
+
if code >= 200 && code < 300
|
|
106
|
+
options[:remote_configuration] = parse_remote_configuration(response)
|
|
107
|
+
return Ports::TransportResult.new(:success, options)
|
|
108
|
+
end
|
|
109
|
+
return Ports::TransportResult.new(:request_timeout, options) if code == 408
|
|
104
110
|
if code == 429
|
|
105
111
|
options[:retry_after] = response["Retry-After"]
|
|
106
112
|
return Ports::TransportResult.new(:rate_limited, options)
|
|
@@ -110,6 +116,18 @@ module Chronos
|
|
|
110
116
|
Ports::TransportResult.new(:client_error, options)
|
|
111
117
|
end
|
|
112
118
|
|
|
119
|
+
def parse_remote_configuration(response)
|
|
120
|
+
return nil unless @config.remote_configuration
|
|
121
|
+
|
|
122
|
+
value = response[REMOTE_CONFIGURATION_HEADER]
|
|
123
|
+
return nil if value.nil? || value.bytesize > @config.remote_config_max_bytes
|
|
124
|
+
|
|
125
|
+
parsed = JSON.parse(value)
|
|
126
|
+
parsed.is_a?(Hash) ? parsed : nil
|
|
127
|
+
rescue JSON::ParserError, EncodingError
|
|
128
|
+
nil
|
|
129
|
+
end
|
|
130
|
+
|
|
113
131
|
def request_body(body)
|
|
114
132
|
return body unless @config.gzip
|
|
115
133
|
|
data/lib/chronos/agent.rb
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
module Chronos
|
|
2
2
|
# Runtime composition root for the framework-independent Chronos agent.
|
|
3
3
|
#
|
|
4
|
-
# @responsibility Own transport,
|
|
4
|
+
# @responsibility Own transport, resilient delivery pipeline, and the capture use case.
|
|
5
5
|
# @motivation Keep construction details outside the public module facade.
|
|
6
6
|
# @limits It does not integrate with Rails, Rack, or job systems.
|
|
7
|
-
# @collaborators Configuration snapshot, CaptureException,
|
|
7
|
+
# @collaborators Configuration snapshot, CaptureException, DeliveryPipeline, and NetHttpTransport.
|
|
8
8
|
# @thread_safety Runtime collaborators synchronize mutable state.
|
|
9
9
|
# @compatibility Ruby 2.2.10 through Ruby 2.6.
|
|
10
10
|
# @example
|
|
@@ -23,9 +23,16 @@ module Chronos
|
|
|
23
23
|
unless Ports::Transport.compatible?(@transport)
|
|
24
24
|
raise ArgumentError, "transport does not implement the Chronos transport port"
|
|
25
25
|
end
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
26
|
+
pipeline_options = {}
|
|
27
|
+
pipeline_options[:queue] = options[:queue] if options[:queue]
|
|
28
|
+
pipeline_options[:worker_pool] = options[:worker_pool] if options[:worker_pool]
|
|
29
|
+
@delivery_pipeline = options[:delivery_pipeline] || Application::DeliveryPipeline.new(
|
|
30
|
+
config,
|
|
31
|
+
@transport,
|
|
32
|
+
@logger,
|
|
33
|
+
pipeline_options
|
|
34
|
+
)
|
|
35
|
+
@capture = options[:capture] || Application::CaptureException.new(config, @delivery_pipeline, @logger)
|
|
29
36
|
end
|
|
30
37
|
|
|
31
38
|
def notify(exception, context = {})
|
|
@@ -37,21 +44,22 @@ module Chronos
|
|
|
37
44
|
end
|
|
38
45
|
|
|
39
46
|
def flush(timeout = DEFAULT_FLUSH_TIMEOUT)
|
|
40
|
-
@
|
|
47
|
+
@delivery_pipeline.flush(timeout)
|
|
41
48
|
rescue StandardError => error
|
|
42
49
|
@logger.warn("Chronos flush failed: #{error.class}")
|
|
43
50
|
false
|
|
44
51
|
end
|
|
45
52
|
|
|
46
53
|
def close(timeout = DEFAULT_FLUSH_TIMEOUT)
|
|
47
|
-
@
|
|
54
|
+
@delivery_pipeline.close(timeout)
|
|
48
55
|
rescue StandardError => error
|
|
49
56
|
@logger.warn("Chronos close failed: #{error.class}")
|
|
50
57
|
false
|
|
51
58
|
end
|
|
52
59
|
|
|
53
60
|
def diagnostics
|
|
54
|
-
@
|
|
61
|
+
details = @delivery_pipeline.diagnostics
|
|
62
|
+
details[:queue].merge(details)
|
|
55
63
|
end
|
|
56
64
|
end
|
|
57
65
|
end
|
|
@@ -2,10 +2,10 @@ module Chronos
|
|
|
2
2
|
module Application
|
|
3
3
|
# Orchestrates exception normalization, serialization, queueing, and delivery.
|
|
4
4
|
#
|
|
5
|
-
# @responsibility Execute the version 0.
|
|
5
|
+
# @responsibility Execute the version 0.3 exception capture pipeline.
|
|
6
6
|
# @motivation Keep the public facade and transport adapters free of use-case logic.
|
|
7
|
-
# @limits It does not implement
|
|
8
|
-
# @collaborators NoticeBuilder, PayloadSerializer,
|
|
7
|
+
# @limits It does not implement framework hooks or automatic capture.
|
|
8
|
+
# @collaborators NoticeBuilder, PayloadSerializer, and DeliveryPipeline.
|
|
9
9
|
# @thread_safety Collaborators are immutable or synchronized; calls may run concurrently.
|
|
10
10
|
# @compatibility Ruby 2.2.10 through Ruby 2.6; independent of Rails.
|
|
11
11
|
# @example
|
|
@@ -13,20 +13,25 @@ module Chronos
|
|
|
13
13
|
# @errors All internal StandardError failures are contained and logged.
|
|
14
14
|
# @performance Local work is bounded before asynchronous queue insertion.
|
|
15
15
|
class CaptureException
|
|
16
|
-
def initialize(config,
|
|
16
|
+
def initialize(config, delivery_pipeline, logger = nil, options = {})
|
|
17
17
|
@config = config
|
|
18
|
-
@
|
|
19
|
-
@transport = transport
|
|
18
|
+
@delivery_pipeline = delivery_pipeline
|
|
20
19
|
@logger = logger || Internal::SafeLogger.new(config.logger)
|
|
21
20
|
@notice_builder = options[:notice_builder] || Core::NoticeBuilder.new(config)
|
|
22
|
-
@serializer = options[:serializer] || Core::PayloadSerializer.new(
|
|
21
|
+
@serializer = options[:serializer] || Core::PayloadSerializer.new(
|
|
22
|
+
config,
|
|
23
|
+
nil,
|
|
24
|
+
:max_payload_size => proc { @delivery_pipeline.max_payload_size }
|
|
25
|
+
)
|
|
23
26
|
end
|
|
24
27
|
|
|
25
28
|
def call(exception, context = {})
|
|
26
29
|
return false unless capture_enabled?
|
|
27
30
|
|
|
28
|
-
|
|
29
|
-
@
|
|
31
|
+
notice = build_notice(exception, context)
|
|
32
|
+
return false unless @delivery_pipeline.capture_allowed?("exception", notice.fingerprint)
|
|
33
|
+
|
|
34
|
+
@delivery_pipeline.enqueue(@serializer.call(notice))
|
|
30
35
|
rescue StandardError => error
|
|
31
36
|
diagnose(error)
|
|
32
37
|
false
|
|
@@ -35,8 +40,10 @@ module Chronos
|
|
|
35
40
|
def call_sync(exception, context = {})
|
|
36
41
|
return false unless capture_enabled?
|
|
37
42
|
|
|
38
|
-
|
|
39
|
-
@
|
|
43
|
+
notice = build_notice(exception, context)
|
|
44
|
+
return false unless @delivery_pipeline.capture_allowed?("exception", notice.fingerprint)
|
|
45
|
+
|
|
46
|
+
@delivery_pipeline.deliver_sync(@serializer.call(notice))
|
|
40
47
|
rescue StandardError => error
|
|
41
48
|
diagnose(error)
|
|
42
49
|
false
|
|
@@ -48,9 +55,8 @@ module Chronos
|
|
|
48
55
|
@config.enabled_for_environment? && @config.error_notifications
|
|
49
56
|
end
|
|
50
57
|
|
|
51
|
-
def
|
|
52
|
-
|
|
53
|
-
@serializer.call(notice)
|
|
58
|
+
def build_notice(exception, context)
|
|
59
|
+
@notice_builder.call(exception, context)
|
|
54
60
|
end
|
|
55
61
|
|
|
56
62
|
def diagnose(error)
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
module Chronos
|
|
2
|
+
module Application
|
|
3
|
+
# Small circuit breaker for retryable transport failures.
|
|
4
|
+
#
|
|
5
|
+
# @responsibility Stop delivery attempts temporarily after repeated failures.
|
|
6
|
+
# @motivation Prevent retry storms while the Chronos endpoint is unavailable.
|
|
7
|
+
# @limits It keeps no payload and permits only one half-open probe.
|
|
8
|
+
# @collaborators DeliveryPipeline and a monotonic clock.
|
|
9
|
+
# @thread_safety All transitions are protected by a mutex.
|
|
10
|
+
# @compatibility Ruby 2.2.10 through Ruby 2.6.
|
|
11
|
+
# @example
|
|
12
|
+
# transport.send_event(event) if breaker.allow_request?
|
|
13
|
+
# @errors Clock failures are contained by DeliveryPipeline.
|
|
14
|
+
# @performance Constant time with fixed memory.
|
|
15
|
+
class CircuitBreaker
|
|
16
|
+
def initialize(failure_threshold, reset_timeout, clock)
|
|
17
|
+
@failure_threshold = failure_threshold
|
|
18
|
+
@reset_timeout = reset_timeout
|
|
19
|
+
@clock = clock
|
|
20
|
+
@state = :closed
|
|
21
|
+
@failures = 0
|
|
22
|
+
@opened_at = nil
|
|
23
|
+
@probe_in_flight = false
|
|
24
|
+
@mutex = Mutex.new
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def allow_request?
|
|
28
|
+
@mutex.synchronize do
|
|
29
|
+
return true if @state == :closed
|
|
30
|
+
return false if @state == :half_open && @probe_in_flight
|
|
31
|
+
return false unless reset_elapsed?
|
|
32
|
+
|
|
33
|
+
@state = :half_open
|
|
34
|
+
@probe_in_flight = true
|
|
35
|
+
true
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def record_success
|
|
40
|
+
@mutex.synchronize do
|
|
41
|
+
@state = :closed
|
|
42
|
+
@failures = 0
|
|
43
|
+
@opened_at = nil
|
|
44
|
+
@probe_in_flight = false
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def record_failure
|
|
49
|
+
@mutex.synchronize do
|
|
50
|
+
@failures += 1
|
|
51
|
+
if @state == :half_open || @failures >= @failure_threshold
|
|
52
|
+
@state = :open
|
|
53
|
+
@opened_at = @clock.call
|
|
54
|
+
end
|
|
55
|
+
@probe_in_flight = false
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def state
|
|
60
|
+
@mutex.synchronize { @state }
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
private
|
|
64
|
+
|
|
65
|
+
def reset_elapsed?
|
|
66
|
+
@opened_at && (@clock.call - @opened_at >= @reset_timeout)
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
module Chronos
|
|
2
|
+
module Application
|
|
3
|
+
# Coordinates bounded queueing, retry, backlog, circuit state, and delivery.
|
|
4
|
+
#
|
|
5
|
+
# @responsibility Move serialized events through explicit delivery states.
|
|
6
|
+
# @motivation Keep resilience policy out of transports, workers, and the public facade.
|
|
7
|
+
# @limits It stores only sanitized SerializedEvent objects and never writes to disk.
|
|
8
|
+
# @collaborators RetryPolicy, CircuitBreaker, RemoteConfiguration, WorkerPool, and Transport.
|
|
9
|
+
# @thread_safety Mutable state, backlog, queue, and circuit transitions are synchronized.
|
|
10
|
+
# @compatibility Ruby 2.2.10 through Ruby 2.6; independent of frameworks.
|
|
11
|
+
# @example
|
|
12
|
+
# pipeline.enqueue(serialized_event)
|
|
13
|
+
# pipeline.flush(2.0)
|
|
14
|
+
# @errors Transport and policy failures are contained and diagnosed.
|
|
15
|
+
# @performance Queue, retry count, backlog, and state counters are strictly bounded.
|
|
16
|
+
class DeliveryPipeline
|
|
17
|
+
STATES = [:accepted, :queued, :serialized, :sent, :retried, :dropped, :rejected].freeze
|
|
18
|
+
|
|
19
|
+
attr_reader :remote_configuration
|
|
20
|
+
|
|
21
|
+
def initialize(config, transport, logger = nil, options = {})
|
|
22
|
+
@config = config
|
|
23
|
+
@transport = transport
|
|
24
|
+
@logger = logger || Internal::SafeLogger.new(config.logger)
|
|
25
|
+
initialize_resilience(options)
|
|
26
|
+
initialize_storage(options)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def enqueue(event)
|
|
30
|
+
record_pre_delivery(event)
|
|
31
|
+
if @worker_pool.enqueue(event)
|
|
32
|
+
transition(:queued)
|
|
33
|
+
true
|
|
34
|
+
else
|
|
35
|
+
transition(:dropped)
|
|
36
|
+
false
|
|
37
|
+
end
|
|
38
|
+
rescue StandardError => error
|
|
39
|
+
diagnose(error)
|
|
40
|
+
transition(:dropped)
|
|
41
|
+
false
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def deliver_sync(event)
|
|
45
|
+
record_pre_delivery(event)
|
|
46
|
+
result = deliver_with_backlog(event)
|
|
47
|
+
result && result.success?
|
|
48
|
+
rescue StandardError => error
|
|
49
|
+
diagnose(error)
|
|
50
|
+
store_in_backlog(event)
|
|
51
|
+
false
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# WorkerPool delivery entry point. Events were counted when accepted into the queue.
|
|
55
|
+
def send_event(event)
|
|
56
|
+
deliver_with_backlog(event)
|
|
57
|
+
rescue StandardError => error
|
|
58
|
+
diagnose(error)
|
|
59
|
+
store_in_backlog(event)
|
|
60
|
+
Ports::TransportResult.new(:network_error, :error => error.class.name)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def capture_allowed?(event_type, fingerprint = nil)
|
|
64
|
+
@remote_configuration.capture?(event_type, fingerprint)
|
|
65
|
+
rescue StandardError => error
|
|
66
|
+
diagnose(error)
|
|
67
|
+
false
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def max_payload_size
|
|
71
|
+
@remote_configuration.max_payload_size
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def flush(timeout)
|
|
75
|
+
@worker_pool.flush(timeout)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def close(timeout)
|
|
79
|
+
return true if closed?
|
|
80
|
+
|
|
81
|
+
@state_mutex.synchronize { @closed = true }
|
|
82
|
+
flushed = @worker_pool.close(timeout)
|
|
83
|
+
@transport.close
|
|
84
|
+
flushed
|
|
85
|
+
rescue StandardError => error
|
|
86
|
+
diagnose(error)
|
|
87
|
+
false
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def diagnostics
|
|
91
|
+
{
|
|
92
|
+
:states => state_counts,
|
|
93
|
+
:queue => @queue.stats,
|
|
94
|
+
:backlog => @backlog.stats,
|
|
95
|
+
:circuit => @circuit_breaker.state,
|
|
96
|
+
:remote_configuration => @remote_configuration.to_h
|
|
97
|
+
}
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
private
|
|
101
|
+
|
|
102
|
+
def initialize_resilience(options)
|
|
103
|
+
@clock = options[:clock] || method(:monotonic_time)
|
|
104
|
+
@sleeper = options[:sleeper] || proc { |delay| sleep(delay) }
|
|
105
|
+
@remote_configuration = options[:remote_configuration] || RemoteConfiguration.new(@config)
|
|
106
|
+
@retry_policy = options[:retry_policy] || build_retry_policy(options[:random])
|
|
107
|
+
@circuit_breaker = options[:circuit_breaker] || CircuitBreaker.new(
|
|
108
|
+
@config.circuit_failure_threshold,
|
|
109
|
+
@config.circuit_reset_timeout,
|
|
110
|
+
@clock
|
|
111
|
+
)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def initialize_storage(options)
|
|
115
|
+
@backlog = options[:backlog] || Internal::MemoryBacklog.new(@config.backlog_size)
|
|
116
|
+
@queue = options[:queue] || Internal::BoundedQueue.new(@config.queue_size)
|
|
117
|
+
@state_mutex = Mutex.new
|
|
118
|
+
@states = STATES.each_with_object({}) { |state, counts| counts[state] = 0 }
|
|
119
|
+
@send_mutex = Mutex.new
|
|
120
|
+
@last_send_at = nil
|
|
121
|
+
@closed = false
|
|
122
|
+
@worker_pool = options[:worker_pool] || Internal::WorkerPool.new(@queue, self, @config.workers, @logger)
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def build_retry_policy(random)
|
|
126
|
+
RetryPolicy.new(
|
|
127
|
+
:max_retries => @config.max_retries,
|
|
128
|
+
:base_interval => @config.retry_base_interval,
|
|
129
|
+
:max_interval => @config.retry_max_interval,
|
|
130
|
+
:jitter => @config.retry_jitter,
|
|
131
|
+
:random => random
|
|
132
|
+
)
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def record_pre_delivery(event)
|
|
136
|
+
raise ArgumentError, "delivery requires a SerializedEvent" unless event.is_a?(Core::SerializedEvent)
|
|
137
|
+
raise Error, "delivery pipeline is closed" if closed?
|
|
138
|
+
|
|
139
|
+
transition(:accepted)
|
|
140
|
+
transition(:serialized)
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def deliver_with_backlog(event)
|
|
144
|
+
drain_one_backlog unless @backlog.empty?
|
|
145
|
+
attempt_delivery(event)
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def drain_one_backlog
|
|
149
|
+
event = @backlog.shift
|
|
150
|
+
attempt_delivery(event) if event
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def attempt_delivery(event)
|
|
154
|
+
unless @circuit_breaker.allow_request?
|
|
155
|
+
transition(:retried)
|
|
156
|
+
store_in_backlog(event)
|
|
157
|
+
return Ports::TransportResult.new(:circuit_open)
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
retries = 0
|
|
161
|
+
loop do
|
|
162
|
+
wait_for_send_interval
|
|
163
|
+
result = @transport.send_event(event)
|
|
164
|
+
return finish_success(result) if result.success?
|
|
165
|
+
return finish_permanent_failure(result) unless result.retryable?
|
|
166
|
+
|
|
167
|
+
@circuit_breaker.record_failure
|
|
168
|
+
unless @retry_policy.retry?(result, retries) && @circuit_breaker.allow_request?
|
|
169
|
+
store_in_backlog(event)
|
|
170
|
+
return result
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
retries += 1
|
|
174
|
+
transition(:retried)
|
|
175
|
+
@sleeper.call(@retry_policy.delay(retries, result))
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def finish_success(result)
|
|
180
|
+
@circuit_breaker.record_success
|
|
181
|
+
apply_remote_configuration(result)
|
|
182
|
+
transition(:sent)
|
|
183
|
+
result
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
def finish_permanent_failure(result)
|
|
187
|
+
@circuit_breaker.record_success
|
|
188
|
+
transition(result.status == :client_error ? :rejected : :dropped)
|
|
189
|
+
result
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def store_in_backlog(event)
|
|
193
|
+
accepted = @backlog.push(event)
|
|
194
|
+
transition(:dropped) unless accepted
|
|
195
|
+
accepted
|
|
196
|
+
rescue StandardError => error
|
|
197
|
+
diagnose(error)
|
|
198
|
+
transition(:dropped)
|
|
199
|
+
false
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def apply_remote_configuration(result)
|
|
203
|
+
values = result.remote_configuration
|
|
204
|
+
return unless values && @config.remote_configuration
|
|
205
|
+
|
|
206
|
+
@remote_configuration.apply(values)
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def wait_for_send_interval
|
|
210
|
+
interval = @remote_configuration.send_interval
|
|
211
|
+
return if interval <= 0.0
|
|
212
|
+
|
|
213
|
+
@send_mutex.synchronize do
|
|
214
|
+
now = @clock.call
|
|
215
|
+
if @last_send_at
|
|
216
|
+
remaining = interval - (now - @last_send_at)
|
|
217
|
+
@sleeper.call(remaining) if remaining > 0.0
|
|
218
|
+
end
|
|
219
|
+
@last_send_at = @clock.call
|
|
220
|
+
end
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
def transition(state)
|
|
224
|
+
@state_mutex.synchronize { @states[state] += 1 }
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
def state_counts
|
|
228
|
+
@state_mutex.synchronize { @states.dup }
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def closed?
|
|
232
|
+
@state_mutex.synchronize { @closed }
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
def diagnose(error)
|
|
236
|
+
@logger.warn("Chronos delivery failed: #{error.class}")
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
def monotonic_time
|
|
240
|
+
if Process.respond_to?(:clock_gettime) && defined?(Process::CLOCK_MONOTONIC)
|
|
241
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
242
|
+
else
|
|
243
|
+
Time.now.to_f
|
|
244
|
+
end
|
|
245
|
+
end
|
|
246
|
+
end
|
|
247
|
+
end
|
|
248
|
+
end
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
module Chronos
|
|
2
|
+
module Application
|
|
3
|
+
# Runtime policy restricted to server-authorized scalar options.
|
|
4
|
+
#
|
|
5
|
+
# @responsibility Validate and apply bounded remote delivery controls.
|
|
6
|
+
# @motivation Let operators reduce telemetry safely without redeploying an application.
|
|
7
|
+
# @limits It cannot change host, credentials, TLS, code, or regular expressions.
|
|
8
|
+
# @collaborators Configuration snapshot, CaptureException, and DeliveryPipeline.
|
|
9
|
+
# @thread_safety A mutex protects immutable state replacements.
|
|
10
|
+
# @compatibility Ruby 2.2.10 through Ruby 2.6.
|
|
11
|
+
# @example
|
|
12
|
+
# remote.apply("sampling_rate" => 0.5, "kill_switch" => false)
|
|
13
|
+
# @errors Invalid documents are ignored and return false.
|
|
14
|
+
# @performance Validation is bounded by configured document and list limits.
|
|
15
|
+
class RemoteConfiguration
|
|
16
|
+
SUPPORTED_EVENT_TYPES = ["exception"].freeze
|
|
17
|
+
MAX_IGNORED_FINGERPRINTS = 100
|
|
18
|
+
MAX_FINGERPRINT_BYTES = 256
|
|
19
|
+
MIN_PAYLOAD_SIZE = 256
|
|
20
|
+
|
|
21
|
+
attr_reader :ignored_fingerprints
|
|
22
|
+
|
|
23
|
+
def initialize(config, options = {})
|
|
24
|
+
@config = config
|
|
25
|
+
@random = options[:random] || proc { rand }
|
|
26
|
+
@mutex = Mutex.new
|
|
27
|
+
@local_event_types = allowed_event_types(config.enabled_event_types)
|
|
28
|
+
@values = {
|
|
29
|
+
"sampling_rate" => config.sampling_rate.to_f,
|
|
30
|
+
"enabled_event_types" => @local_event_types,
|
|
31
|
+
"max_payload_size" => config.max_payload_size,
|
|
32
|
+
"ignored_fingerprints" => [],
|
|
33
|
+
"send_interval" => 0.0,
|
|
34
|
+
"kill_switch" => false
|
|
35
|
+
}
|
|
36
|
+
@ignored_fingerprints = [].freeze
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def apply(document)
|
|
40
|
+
normalized = normalize(document)
|
|
41
|
+
return false unless normalized
|
|
42
|
+
|
|
43
|
+
@mutex.synchronize do
|
|
44
|
+
@values = @values.merge(normalized)
|
|
45
|
+
@ignored_fingerprints = @values["ignored_fingerprints"].dup.freeze
|
|
46
|
+
end
|
|
47
|
+
true
|
|
48
|
+
rescue StandardError
|
|
49
|
+
false
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def capture?(event_type, fingerprint = nil)
|
|
53
|
+
values = snapshot
|
|
54
|
+
return false if values["kill_switch"]
|
|
55
|
+
return false unless values["enabled_event_types"].include?(event_type.to_s)
|
|
56
|
+
return false if fingerprint && values["ignored_fingerprints"].include?(fingerprint.to_s)
|
|
57
|
+
|
|
58
|
+
rate = values["sampling_rate"]
|
|
59
|
+
rate >= 1.0 || (rate > 0.0 && @random.call.to_f < rate)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def enabled_event?(event_type)
|
|
63
|
+
snapshot["enabled_event_types"].include?(event_type.to_s)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def ignored_fingerprint?(fingerprint)
|
|
67
|
+
snapshot["ignored_fingerprints"].include?(fingerprint.to_s)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def sampling_rate
|
|
71
|
+
snapshot["sampling_rate"]
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def max_payload_size
|
|
75
|
+
snapshot["max_payload_size"]
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def send_interval
|
|
79
|
+
snapshot["send_interval"]
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def kill_switch?
|
|
83
|
+
snapshot["kill_switch"]
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def to_h
|
|
87
|
+
snapshot
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
private
|
|
91
|
+
|
|
92
|
+
def normalize(document)
|
|
93
|
+
return nil unless document.is_a?(Hash)
|
|
94
|
+
|
|
95
|
+
normalized = {}
|
|
96
|
+
document.each do |key, value|
|
|
97
|
+
name = key.to_s
|
|
98
|
+
next unless @values.key?(name)
|
|
99
|
+
|
|
100
|
+
normalized_value = normalize_value(name, value)
|
|
101
|
+
return nil if normalized_value.nil? && value != false
|
|
102
|
+
normalized[name] = normalized_value
|
|
103
|
+
end
|
|
104
|
+
normalized.empty? ? nil : normalized
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def normalize_value(name, value)
|
|
108
|
+
case name
|
|
109
|
+
when "sampling_rate"
|
|
110
|
+
bounded_rate(value)
|
|
111
|
+
when "enabled_event_types"
|
|
112
|
+
normalize_event_types(value)
|
|
113
|
+
when "max_payload_size"
|
|
114
|
+
normalize_payload_size(value)
|
|
115
|
+
when "ignored_fingerprints"
|
|
116
|
+
normalize_fingerprints(value)
|
|
117
|
+
when "send_interval"
|
|
118
|
+
normalize_send_interval(value)
|
|
119
|
+
when "kill_switch"
|
|
120
|
+
value if [true, false].include?(value)
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def bounded_rate(value)
|
|
125
|
+
return nil unless value.is_a?(Numeric) && value >= 0.0 && value <= @config.sampling_rate
|
|
126
|
+
|
|
127
|
+
value.to_f
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def normalize_event_types(value)
|
|
131
|
+
return nil unless value.is_a?(Array) && value.all? { |item| item.is_a?(String) }
|
|
132
|
+
|
|
133
|
+
allowed_event_types(value)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def allowed_event_types(values)
|
|
137
|
+
supported = values.map(&:to_s).select { |value| SUPPORTED_EVENT_TYPES.include?(value) }
|
|
138
|
+
supported.select { |value| local_event_types.include?(value) }.uniq.freeze
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def local_event_types
|
|
142
|
+
@local_event_types || SUPPORTED_EVENT_TYPES
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def normalize_payload_size(value)
|
|
146
|
+
return nil unless value.is_a?(Integer) && value >= MIN_PAYLOAD_SIZE && value <= @config.max_payload_size
|
|
147
|
+
|
|
148
|
+
value
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def normalize_fingerprints(value)
|
|
152
|
+
return nil unless value.is_a?(Array) && value.length <= MAX_IGNORED_FINGERPRINTS
|
|
153
|
+
return nil unless value.all? { |item| item.is_a?(String) && item.bytesize <= MAX_FINGERPRINT_BYTES }
|
|
154
|
+
|
|
155
|
+
value.map(&:dup).freeze
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def normalize_send_interval(value)
|
|
159
|
+
return nil unless value.is_a?(Numeric) && value >= 0.0 && value <= @config.max_remote_send_interval
|
|
160
|
+
|
|
161
|
+
value.to_f
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def snapshot
|
|
165
|
+
@mutex.synchronize do
|
|
166
|
+
@values.each_with_object({}) do |(key, value), copy|
|
|
167
|
+
copy[key] = value.is_a?(Array) ? value.dup : value
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
end
|