sbmt-kafka_consumer 2.4.0 → 2.5.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 3aaf5a3d0cab0fe477ab5af189b754b642c403036eec6f7329818645a1dff362
4
- data.tar.gz: f198d12579b3437f903fff1c7be33246c5121b1b37bb735e861c7c0f9cbb0314
3
+ metadata.gz: 6e41656d16c2a1b741868c2d079e4e83c3df397e5e7491680e417738231afe71
4
+ data.tar.gz: '09117d5de7194d45bf44235f731adfd4e6f60cc18d6c613c1fe2a3800048a89c'
5
5
  SHA512:
6
- metadata.gz: 468be419229e1096fbae7a0892607c33c8913f686478bca5d8ed8816ac83c8932be7683eedbfbc8eeb55225578dd74707c7bc256947691e65fa19d7142fdb095
7
- data.tar.gz: 105b7fdd0b37927b92e81819035442705e9a636828469f47b61ce5bb67850db2f1c921e5d45b9bcadedb89eaceb6e929a3fde903672517a9aae5810e68ef0627
6
+ metadata.gz: 0ce2e28e58baf91e4e1612d02c37767399e643ed99c78f26adfcae1e23e77aab8f1af8bda64c6f91b16456e069b24f7db53812843347924b124b9e890d8f4e69
7
+ data.tar.gz: 3447e5c1c06c3f636d4c5f37070a9f349714aea586710ba7c87d2cd3248ba5ad686bfca28d1550ec0af86d18cd556315945596c7b7c069fa31a4d0db2e1425b9
data/CHANGELOG.md CHANGED
@@ -13,6 +13,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
13
13
 
14
14
  ### Fixed
15
15
 
16
+ ## [2.5.0] - 2024-06-24
17
+
18
+ ### Added
19
+
20
+ - Added option `max_error_count` for liveness probes, which is triggered when `librdkafka.error`
21
+
22
+ ## [2.4.1] - 2024-06-15
23
+
24
+ ### Fixed
25
+
26
+ - Fixed display of metrics `kafka_api_calls` and `kafka_api_errors`
27
+
16
28
  ## [2.4.0] - 2024-06-06
17
29
 
18
30
  ### Added
data/README.md CHANGED
@@ -92,6 +92,7 @@ default: &default
92
92
  enabled: true
93
93
  path: "/liveness"
94
94
  timeout: 15
95
+ max_error_count: 15 # default 10
95
96
  metrics: # optional section
96
97
  port: 9090
97
98
  path: "/metrics"
@@ -8,4 +8,5 @@ class Sbmt::KafkaConsumer::Config::Probes::LivenessProbe < Dry::Struct
8
8
  .optional
9
9
  .default("/liveness")
10
10
  attribute :timeout, Sbmt::KafkaConsumer::Types::Coercible::Integer.optional.default(10)
11
+ attribute :max_error_count, Sbmt::KafkaConsumer::Types::Coercible::Integer.optional.default(10)
11
12
  end
@@ -7,9 +7,14 @@ module Sbmt
7
7
  include ListenerHelper
8
8
  include KafkaConsumer::Probes::Probe
9
9
 
10
- def initialize(timeout_sec: 10)
10
+ ERROR_TYPE = "Liveness probe error"
11
+
12
+ def initialize(timeout_sec: 10, max_error_count: 10)
11
13
  @consumer_groups = Karafka::App.routes.map(&:name)
12
14
  @timeout_sec = timeout_sec
15
+ @max_error_count = max_error_count
16
+ @error_count = 0
17
+ @error_backtrace = nil
13
18
  @polls = {}
14
19
 
15
20
  setup_subscription
@@ -18,9 +23,14 @@ module Sbmt
18
23
  def probe(_env)
19
24
  now = current_time
20
25
  timed_out_polls = select_timed_out_polls(now)
21
- return probe_ok groups: meta_from_polls(polls, now) if timed_out_polls.empty?
22
26
 
23
- probe_error failed_groups: meta_from_polls(timed_out_polls, now)
27
+ if timed_out_polls.empty? && @error_count < @max_error_count
28
+ probe_ok groups: meta_from_polls(polls, now) if timed_out_polls.empty?
29
+ elsif @error_count >= @max_error_count
30
+ probe_error error_type: ERROR_TYPE, failed_librdkafka: {error_count: @error_count, error_backtrace: @error_backtrace}
31
+ else
32
+ probe_error error_type: ERROR_TYPE, failed_groups: meta_from_polls(timed_out_polls, now)
33
+ end
24
34
  end
25
35
 
26
36
  def on_connection_listener_fetch_loop(event)
@@ -28,6 +38,16 @@ module Sbmt
28
38
  polls[consumer_group.name] = current_time
29
39
  end
30
40
 
41
+ def on_error_occurred(event)
42
+ type = event[:type]
43
+
44
+ return unless type == "librdkafka.error"
45
+ error = event[:error]
46
+
47
+ @error_backtrace ||= (error.backtrace || []).join("\n")
48
+ @error_count += 1
49
+ end
50
+
31
51
  private
32
52
 
33
53
  attr_reader :polls, :timeout_sec, :consumer_groups
@@ -129,7 +129,7 @@ module Sbmt
129
129
  }
130
130
 
131
131
  Yabeda.kafka_api.calls
132
- .increment(broker_tags, by: broker_statistics["tx"])
132
+ .increment(broker_tags, by: broker_statistics["tx_d"])
133
133
  Yabeda.kafka_api.latency
134
134
  .measure(broker_tags, broker_statistics["rtt"]["avg"])
135
135
  Yabeda.kafka_api.request_size
@@ -137,7 +137,7 @@ module Sbmt
137
137
  Yabeda.kafka_api.response_size
138
138
  .measure(broker_tags, broker_statistics["rxbytes"])
139
139
  Yabeda.kafka_api.errors
140
- .increment(broker_tags, by: broker_statistics["txerrs"] + broker_statistics["rxerrs"])
140
+ .increment(broker_tags, by: broker_statistics["txerrs_d"] + broker_statistics["rxerrs_d"])
141
141
  end
142
142
  end
143
143
 
@@ -23,7 +23,7 @@ module Sbmt
23
23
  liveness = config[:liveness]
24
24
  if liveness[:enabled]
25
25
  c.probe liveness[:path], Sbmt::KafkaConsumer::Instrumentation::LivenessListener.new(
26
- timeout_sec: liveness[:timeout]
26
+ timeout_sec: liveness[:timeout], max_error_count: liveness[:max_error_count]
27
27
  )
28
28
  end
29
29
 
@@ -19,6 +19,7 @@ module Sbmt
19
19
  end
20
20
 
21
21
  def probe_error(extra_meta = {})
22
+ KafkaConsumer.logger.error("probe error meta: #{meta.merge(extra_meta).inspect}")
22
23
  [500, HEADERS, [meta.merge(extra_meta).to_json]]
23
24
  end
24
25
 
@@ -2,6 +2,6 @@
2
2
 
3
3
  module Sbmt
4
4
  module KafkaConsumer
5
- VERSION = "2.4.0"
5
+ VERSION = "2.5.0"
6
6
  end
7
7
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sbmt-kafka_consumer
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.4.0
4
+ version: 2.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sbermarket Ruby-Platform Team
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-06-07 00:00:00.000000000 Z
11
+ date: 2024-06-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rails
@@ -570,7 +570,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
570
570
  - !ruby/object:Gem::Version
571
571
  version: '0'
572
572
  requirements: []
573
- rubygems_version: 3.5.11
573
+ rubygems_version: 3.1.6
574
574
  signing_key:
575
575
  specification_version: 4
576
576
  summary: Ruby gem for consuming Kafka messages