ruby-kafka 1.1.0.beta1 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -47,18 +47,19 @@ module Kafka
47
47
  Array(ca_cert).each do |cert|
48
48
  store.add_cert(OpenSSL::X509::Certificate.new(cert))
49
49
  end
50
- if ca_cert_file_path
51
- store.add_file(ca_cert_file_path)
50
+ Array(ca_cert_file_path).each do |cert_file_path|
51
+ store.add_file(cert_file_path)
52
52
  end
53
53
  if ca_certs_from_system
54
54
  store.set_default_paths
55
55
  end
56
56
  ssl_context.cert_store = store
57
- ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
58
- # Verify certificate hostname if supported (ruby >= 2.4.0)
59
- ssl_context.verify_hostname = verify_hostname if ssl_context.respond_to?(:verify_hostname=)
60
57
  end
61
58
 
59
+ ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
60
+ # Verify certificate hostname if supported (ruby >= 2.4.0)
61
+ ssl_context.verify_hostname = verify_hostname if ssl_context.respond_to?(:verify_hostname=)
62
+
62
63
  ssl_context
63
64
  end
64
65
  end
@@ -95,7 +95,7 @@ module Kafka
95
95
  force_transactional!
96
96
 
97
97
  if @transaction_state.uninitialized?
98
- raise 'Transaction is uninitialized'
98
+ raise Kafka::InvalidTxnStateError, 'Transaction is uninitialized'
99
99
  end
100
100
 
101
101
  # Extract newly created partitions
@@ -138,8 +138,8 @@ module Kafka
138
138
 
139
139
  def begin_transaction
140
140
  force_transactional!
141
- raise 'Transaction has already started' if @transaction_state.in_transaction?
142
- raise 'Transaction is not ready' unless @transaction_state.ready?
141
+ raise Kafka::InvalidTxnStateError, 'Transaction has already started' if @transaction_state.in_transaction?
142
+ raise Kafka::InvalidTxnStateError, 'Transaction is not ready' unless @transaction_state.ready?
143
143
  @transaction_state.transition_to!(TransactionStateMachine::IN_TRANSACTION)
144
144
 
145
145
  @logger.info "Begin transaction #{@transactional_id}, Producer ID: #{@producer_id} (Epoch #{@producer_epoch})"
@@ -159,7 +159,7 @@ module Kafka
159
159
  end
160
160
 
161
161
  unless @transaction_state.in_transaction?
162
- raise 'Transaction is not valid to commit'
162
+ raise Kafka::InvalidTxnStateError, 'Transaction is not valid to commit'
163
163
  end
164
164
 
165
165
  @transaction_state.transition_to!(TransactionStateMachine::COMMITTING_TRANSACTION)
@@ -192,7 +192,8 @@ module Kafka
192
192
  end
193
193
 
194
194
  unless @transaction_state.in_transaction?
195
- raise 'Transaction is not valid to abort'
195
+ @logger.warn('Aborting transaction that was never opened on brokers')
196
+ return
196
197
  end
197
198
 
198
199
  @transaction_state.transition_to!(TransactionStateMachine::ABORTING_TRANSACTION)
@@ -221,7 +222,7 @@ module Kafka
221
222
  force_transactional!
222
223
 
223
224
  unless @transaction_state.in_transaction?
224
- raise 'Transaction is not valid to send offsets'
225
+ raise Kafka::InvalidTxnStateError, 'Transaction is not valid to send offsets'
225
226
  end
226
227
 
227
228
  add_response = transaction_coordinator.add_offsets_to_txn(
@@ -232,14 +233,23 @@ module Kafka
232
233
  )
233
234
  Protocol.handle_error(add_response.error_code)
234
235
 
235
- send_response = transaction_coordinator.txn_offset_commit(
236
+ send_response = group_coordinator(group_id: group_id).txn_offset_commit(
236
237
  transactional_id: @transactional_id,
237
238
  group_id: group_id,
238
239
  producer_id: @producer_id,
239
240
  producer_epoch: @producer_epoch,
240
241
  offsets: offsets
241
242
  )
242
- Protocol.handle_error(send_response.error_code)
243
+ send_response.errors.each do |tp|
244
+ tp.partitions.each do |partition|
245
+ Protocol.handle_error(partition.error_code)
246
+ end
247
+ end
248
+
249
+ nil
250
+ rescue
251
+ @transaction_state.transition_to!(TransactionStateMachine::ERROR)
252
+ raise
243
253
  end
244
254
 
245
255
  def in_transaction?
@@ -250,6 +260,10 @@ module Kafka
250
260
  @transaction_state.error?
251
261
  end
252
262
 
263
+ def ready?
264
+ @transaction_state.ready?
265
+ end
266
+
253
267
  def close
254
268
  if in_transaction?
255
269
  @logger.warn("Aborting pending transaction ...")
@@ -264,11 +278,11 @@ module Kafka
264
278
 
265
279
  def force_transactional!
266
280
  unless transactional?
267
- raise 'Please turn on transactional mode to use transaction'
281
+ raise Kafka::InvalidTxnStateError, 'Please turn on transactional mode to use transaction'
268
282
  end
269
283
 
270
284
  if @transactional_id.nil? || @transactional_id.empty?
271
- raise 'Please provide a transaction_id to use transactional mode'
285
+ raise Kafka::InvalidTxnStateError, 'Please provide a transaction_id to use transactional mode'
272
286
  end
273
287
  end
274
288
 
@@ -278,6 +292,12 @@ module Kafka
278
292
  )
279
293
  end
280
294
 
295
+ def group_coordinator(group_id:)
296
+ @cluster.get_group_coordinator(
297
+ group_id: group_id
298
+ )
299
+ end
300
+
281
301
  def complete_transaction
282
302
  @transaction_state.transition_to!(TransactionStateMachine::READY)
283
303
  @transaction_partitions = {}
data/lib/kafka/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Kafka
4
- VERSION = "1.1.0.beta1"
4
+ VERSION = "1.4.0"
5
5
  end
data/ruby-kafka.gemspec CHANGED
@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
18
18
  DESC
19
19
 
20
20
  spec.homepage = "https://github.com/zendesk/ruby-kafka"
21
- spec.license = "Apache License Version 2.0"
21
+ spec.license = "Apache-2.0"
22
22
 
23
23
  spec.required_ruby_version = '>= 2.1.0'
24
24
 
@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
33
33
  spec.add_development_dependency "rake", "~> 10.0"
34
34
  spec.add_development_dependency "rspec"
35
35
  spec.add_development_dependency "pry"
36
+ spec.add_development_dependency "digest-murmurhash"
36
37
  spec.add_development_dependency "dotenv"
37
38
  spec.add_development_dependency "docker-api"
38
39
  spec.add_development_dependency "rspec-benchmark"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ruby-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.0.beta1
4
+ version: 1.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Daniel Schierbeck
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-05-05 00:00:00.000000000 Z
11
+ date: 2021-08-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: digest-crc
@@ -80,6 +80,20 @@ dependencies:
80
80
  - - ">="
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: digest-murmurhash
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
83
97
  - !ruby/object:Gem::Dependency
84
98
  name: dotenv
85
99
  requirement: !ruby/object:Gem::Requirement
@@ -376,7 +390,10 @@ files:
376
390
  - lib/kafka/connection_builder.rb
377
391
  - lib/kafka/consumer.rb
378
392
  - lib/kafka/consumer_group.rb
393
+ - lib/kafka/consumer_group/assignor.rb
394
+ - lib/kafka/crc32_hash.rb
379
395
  - lib/kafka/datadog.rb
396
+ - lib/kafka/digest.rb
380
397
  - lib/kafka/fetch_operation.rb
381
398
  - lib/kafka/fetched_batch.rb
382
399
  - lib/kafka/fetched_batch_generator.rb
@@ -386,8 +403,10 @@ files:
386
403
  - lib/kafka/gzip_codec.rb
387
404
  - lib/kafka/heartbeat.rb
388
405
  - lib/kafka/instrumenter.rb
406
+ - lib/kafka/interceptors.rb
389
407
  - lib/kafka/lz4_codec.rb
390
408
  - lib/kafka/message_buffer.rb
409
+ - lib/kafka/murmur2_hash.rb
391
410
  - lib/kafka/offset_manager.rb
392
411
  - lib/kafka/partitioner.rb
393
412
  - lib/kafka/pause.rb
@@ -476,7 +495,7 @@ files:
476
495
  - ruby-kafka.gemspec
477
496
  homepage: https://github.com/zendesk/ruby-kafka
478
497
  licenses:
479
- - Apache License Version 2.0
498
+ - Apache-2.0
480
499
  metadata: {}
481
500
  post_install_message:
482
501
  rdoc_options: []
@@ -489,12 +508,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
489
508
  version: 2.1.0
490
509
  required_rubygems_version: !ruby/object:Gem::Requirement
491
510
  requirements:
492
- - - ">"
511
+ - - ">="
493
512
  - !ruby/object:Gem::Version
494
- version: 1.3.1
513
+ version: '0'
495
514
  requirements: []
496
- rubyforge_project:
497
- rubygems_version: 2.7.6
515
+ rubygems_version: 3.1.2
498
516
  signing_key:
499
517
  specification_version: 4
500
518
  summary: A client library for the Kafka distributed commit log.