ruby-kafka 0.3.13.beta2 → 0.3.13.beta3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/kafka/cluster.rb +31 -13
- data/lib/kafka/offset_manager.rb +14 -2
- data/lib/kafka/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0185da2c30e9d614d899d9c64257ce6e4f3c3623
|
4
|
+
data.tar.gz: 88d010e2aa98c3177e1591db1b89a87603c7872d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a693bc3483e3e5d621b3f29ac59978fcdaa107dba84c601edbe46a2991158ec32180f88447e5f94ada737463f3fafbd572e4fbd5f0dd5e4e7afcb6d4e17612de
|
7
|
+
data.tar.gz: aeda8eb706dd0a96004f847c0ff6e5ae97a2973849f95b70b80c99f58fa049439fe2bbb51f0a0efcb80bf6526a0a1ca6b61761486304ca458a9101a47a65b4f4
|
data/lib/kafka/cluster.rb
CHANGED
@@ -119,10 +119,16 @@ module Kafka
|
|
119
119
|
raise
|
120
120
|
end
|
121
121
|
|
122
|
-
def
|
122
|
+
def resolve_offsets(topic, partitions, offset)
|
123
123
|
add_target_topics([topic])
|
124
124
|
refresh_metadata_if_necessary!
|
125
|
-
|
125
|
+
|
126
|
+
partitions_by_broker = partitions.each_with_object({}) {|partition, hsh|
|
127
|
+
broker = get_leader(topic, partition)
|
128
|
+
|
129
|
+
hsh[broker] ||= []
|
130
|
+
hsh[broker] << partition
|
131
|
+
}
|
126
132
|
|
127
133
|
if offset == :earliest
|
128
134
|
offset = -2
|
@@ -130,19 +136,31 @@ module Kafka
|
|
130
136
|
offset = -1
|
131
137
|
end
|
132
138
|
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
139
|
+
offsets = {}
|
140
|
+
|
141
|
+
partitions_by_broker.each do |broker, broker_partitions|
|
142
|
+
response = broker.list_offsets(
|
143
|
+
topics: {
|
144
|
+
topic => broker_partitions.map {|partition|
|
145
|
+
{
|
146
|
+
partition: partition,
|
147
|
+
time: offset,
|
148
|
+
max_offsets: 1,
|
149
|
+
}
|
140
150
|
}
|
141
|
-
|
142
|
-
|
143
|
-
)
|
151
|
+
}
|
152
|
+
)
|
144
153
|
|
145
|
-
|
154
|
+
broker_partitions.each do |partition|
|
155
|
+
offsets[partition] = response.offset_for(topic, partition)
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
offsets
|
160
|
+
end
|
161
|
+
|
162
|
+
def resolve_offset(topic, partition, offset)
|
163
|
+
resolve_offsets(topic, [partition], offset).fetch(partition)
|
146
164
|
end
|
147
165
|
|
148
166
|
def topics
|
data/lib/kafka/offset_manager.rb
CHANGED
@@ -11,6 +11,7 @@ module Kafka
|
|
11
11
|
@processed_offsets = {}
|
12
12
|
@default_offsets = {}
|
13
13
|
@committed_offsets = nil
|
14
|
+
@resolved_offsets = {}
|
14
15
|
@last_commit = Time.now
|
15
16
|
end
|
16
17
|
|
@@ -33,8 +34,7 @@ module Kafka
|
|
33
34
|
# A negative offset means that no offset has been committed, so we need to
|
34
35
|
# resolve the default offset for the topic.
|
35
36
|
if offset < 0
|
36
|
-
|
37
|
-
@cluster.resolve_offset(topic, partition, default_offset)
|
37
|
+
resolve_offset(topic, partition)
|
38
38
|
else
|
39
39
|
# The next offset is the last offset plus one.
|
40
40
|
offset + 1
|
@@ -85,6 +85,18 @@ module Kafka
|
|
85
85
|
|
86
86
|
private
|
87
87
|
|
88
|
+
def resolve_offset(topic, partition)
|
89
|
+
@resolved_offsets[topic] ||= fetch_resolved_offsets(topic)
|
90
|
+
@resolved_offsets[topic].fetch(partition)
|
91
|
+
end
|
92
|
+
|
93
|
+
def fetch_resolved_offsets(topic)
|
94
|
+
default_offset = @default_offsets.fetch(topic)
|
95
|
+
partitions = @group.assigned_partitions.fetch(topic)
|
96
|
+
|
97
|
+
@cluster.resolve_offsets(topic, partitions, default_offset)
|
98
|
+
end
|
99
|
+
|
88
100
|
def seconds_since_last_commit
|
89
101
|
Time.now - @last_commit
|
90
102
|
end
|
data/lib/kafka/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ruby-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.13.
|
4
|
+
version: 0.3.13.beta3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Schierbeck
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-08-
|
11
|
+
date: 2016-08-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|