ruby-kafka 0.3.13.beta3 → 0.3.13.beta4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0185da2c30e9d614d899d9c64257ce6e4f3c3623
4
- data.tar.gz: 88d010e2aa98c3177e1591db1b89a87603c7872d
3
+ metadata.gz: 74dfff5457403137f170938a453a12e1288fd3f7
4
+ data.tar.gz: 03074ef585e3a46b45b703ed1efa13ef89e32062
5
5
  SHA512:
6
- metadata.gz: a693bc3483e3e5d621b3f29ac59978fcdaa107dba84c601edbe46a2991158ec32180f88447e5f94ada737463f3fafbd572e4fbd5f0dd5e4e7afcb6d4e17612de
7
- data.tar.gz: aeda8eb706dd0a96004f847c0ff6e5ae97a2973849f95b70b80c99f58fa049439fe2bbb51f0a0efcb80bf6526a0a1ca6b61761486304ca458a9101a47a65b4f4
6
+ metadata.gz: 88a1fc91d4d4b6b1a5aa1250e6f9a5b2a178d1bdb3233531d588ba8440072b64282ae17e866e0812ecbc502ff7ad418d3a0925d04ba710767d28f65af63890e0
7
+ data.tar.gz: 5a3a5b05c8d30679c528bf532584946d762a6d8c038f93e62958119db7f79699e503cb7a14164de43d008e9681cf8f6aca14be2f1ed2da0c7d36fdf1b7791573
@@ -4,6 +4,10 @@ Changes and additions to the library will be listed here.
4
4
 
5
5
  ## Unreleased
6
6
 
7
+ ## v0.3.13.beta4
8
+
9
+ - Automatically recover from invalid consumer checkpoints.
10
+
7
11
  ## v0.3.13.beta1
8
12
 
9
13
  - Minimize the number of times messages are reprocessed after a consumer group resync.
@@ -19,6 +19,7 @@ module Kafka
19
19
  end
20
20
 
21
21
  class OffsetOutOfRange < ProtocolError
22
+ attr_accessor :topic, :partition
22
23
  end
23
24
 
24
25
  # The request is for a topic or partition that does not exist on the broker.
@@ -263,6 +263,12 @@ module Kafka
263
263
  end
264
264
 
265
265
  operation.execute
266
+ rescue OffsetOutOfRange => e
267
+ @logger.error "Invalid offset for #{e.topic}/#{e.partition}, resetting to default offset"
268
+
269
+ @offset_manager.seek_to_default(e.topic, e.partition)
270
+
271
+ retry
266
272
  rescue ConnectionError => e
267
273
  @logger.error "Connection error while fetching messages: #{e}"
268
274
 
@@ -113,7 +113,7 @@ module Kafka
113
113
  rescue UnknownMemberId
114
114
  @logger.error "Failed to join group; resetting member id and retrying in 1s..."
115
115
 
116
- @member_id = nil
116
+ @member_id = ""
117
117
  sleep 1
118
118
 
119
119
  retry
@@ -71,6 +71,11 @@ module Kafka
71
71
  fetched_topic.partitions.map {|fetched_partition|
72
72
  begin
73
73
  Protocol.handle_error(fetched_partition.error_code)
74
+ rescue Kafka::OffsetOutOfRange => e
75
+ e.topic = fetched_topic.name
76
+ e.partition = fetched_partition.partition
77
+
78
+ raise e
74
79
  rescue Kafka::Error => e
75
80
  topic = fetched_topic.name
76
81
  partition = fetched_partition.partition
@@ -26,6 +26,11 @@ module Kafka
26
26
  @logger.debug "Marking #{topic}/#{partition}:#{offset} as committed"
27
27
  end
28
28
 
29
+ def seek_to_default(topic, partition)
30
+ @processed_offsets[topic] ||= {}
31
+ @processed_offsets[topic][partition] = -1
32
+ end
33
+
29
34
  def next_offset_for(topic, partition)
30
35
  offset = @processed_offsets.fetch(topic, {}).fetch(partition) {
31
36
  committed_offset_for(topic, partition)
@@ -360,7 +360,10 @@ module Kafka
360
360
  end
361
361
 
362
362
  if failed_messages.any?
363
- @logger.error "Failed to assign partitions to #{failed_messages.count} messages"
363
+ failed_messages.group_by(&:topic).each do |topic, messages|
364
+ @logger.error "Failed to assign partitions to #{messages.count} messages in #{topic}"
365
+ end
366
+
364
367
  @cluster.mark_as_stale!
365
368
  end
366
369
 
@@ -2,6 +2,9 @@ module Kafka
2
2
  class SnappyCodec
3
3
  def initialize
4
4
  require "snappy"
5
+ rescue LoadError
6
+ raise LoadError,
7
+ "Using snappy compression requires adding a dependency on the `snappy` gem to your Gemfile."
5
8
  end
6
9
 
7
10
  def codec_id
@@ -1,3 +1,3 @@
1
1
  module Kafka
2
- VERSION = "0.3.13.beta3"
2
+ VERSION = "0.3.13.beta4"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ruby-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.13.beta3
4
+ version: 0.3.13.beta4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Daniel Schierbeck
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2016-08-03 00:00:00.000000000 Z
11
+ date: 2016-08-05 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler