promiscuous-poseidon_cluster 0.4.1 → 0.4.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 785249eb37555296aa2d93478182c292850413fe
4
- data.tar.gz: 21cf920cade06e1598fee1b47ea19a553a5ab13b
3
+ metadata.gz: b9fc0d8c24c41642e703279f9feeb80e10c788b8
4
+ data.tar.gz: 6c35c729adac40669cbb915542810ab107f7dd07
5
5
  SHA512:
6
- metadata.gz: dc4faf271c725f70d7535e675f97ec27c933b6c63a95c69927738e79bef160e8d92bbcc8229177c25589bdcd5b32dd62af6b082b99d60f8f2a63fe88945758ce
7
- data.tar.gz: 7040138c26ff1c65bd4c2600f16c339c488f214830d3307bb7050523652bc671ab01257fa364a2adfb481f6604d03f98f3a8cea8d56a64a71d7369c8e085fe09
6
+ metadata.gz: 2b7f1477c8fe08586939a90af7a6b7ee4f03a97e59140afb5252d8442f0ad99926bd644848a40fc844116982460a20ded382c0a1ac1493d5c292acb70ca2979a
7
+ data.tar.gz: d7c875b13cce867d3f43e9969419efc6db74b6fd9e97e3e116c16a3e09b18445cd90564eb44224ca657ef773c5c264d97e95e72b44f4cf9c7777a3133d8bd5fb
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- promiscuous-poseidon_cluster (0.4.1)
4
+ poseidon_cluster (0.4.1)
5
5
  poseidon (>= 0.0.5.pre1)
6
6
  zk
7
7
 
@@ -65,7 +65,7 @@ PLATFORMS
65
65
  DEPENDENCIES
66
66
  bundler
67
67
  coveralls
68
- promiscuous-poseidon_cluster!
68
+ poseidon_cluster!
69
69
  rake
70
70
  rspec
71
71
  rspec-its
@@ -110,7 +110,6 @@ class Poseidon::ConsumerGroup
110
110
  @consumers = []
111
111
  @pool = ::Poseidon::BrokerPool.new(id, brokers, options[:socket_timeout_ms])
112
112
  @mutex = Mutex.new
113
- @registered = false
114
113
 
115
114
  register! unless options[:register] == false
116
115
  end
@@ -141,7 +140,7 @@ class Poseidon::ConsumerGroup
141
140
 
142
141
  # @return [Boolean] true if registered
143
142
  def registered?
144
- @registered
143
+ !!zk.children(consumer_path, ignore: :no_node)
145
144
  end
146
145
 
147
146
  # @return [Boolean] true if registration was successful, false if already registered
@@ -155,9 +154,7 @@ class Poseidon::ConsumerGroup
155
154
  zk.create(consumer_path, "{}", {:ephemeral => true})
156
155
  zk.register(registries[:consumer]) { rebalance! }
157
156
 
158
- # Rebalance
159
157
  rebalance!
160
- @registered = true
161
158
  end
162
159
 
163
160
  # Reloads metadata/broker/partition information
@@ -172,7 +169,7 @@ class Poseidon::ConsumerGroup
172
169
  # Closes the consumer group gracefully, only really useful in tests
173
170
  # @api private
174
171
  def close
175
- @mutex.synchronize { release_all! }
172
+ synchronize { release_all! }
176
173
  zk.close
177
174
  end
178
175
 
@@ -196,6 +193,8 @@ class Poseidon::ConsumerGroup
196
193
  zk.set(offset_path(partition), offset.to_s)
197
194
  rescue ZK::Exceptions::NoNode
198
195
  zk.create(offset_path(partition), offset.to_s, {:ignore => :node_exists})
196
+ ensure
197
+ unlock(offset)
199
198
  end
200
199
 
201
200
  # Sorted partitions by broker address (so partitions on the same broker are clustered together)
@@ -234,20 +233,22 @@ class Poseidon::ConsumerGroup
234
233
  #
235
234
  # @api public
236
235
  def checkout(opts = {})
237
- consumer = nil
238
- commit = @mutex.synchronize do
239
- consumer = @consumers.shift
240
- return false unless consumer
236
+ register!
237
+ lock
241
238
 
242
- @consumers.push consumer
243
- yield consumer
244
- end
239
+ @current_consumer = @consumers.shift
240
+ return false unless @current_consumer
241
+
242
+ @consumers.push @current_consumer
243
+ commit = yield @current_consumer
245
244
 
246
245
  unless opts[:commit] == false || commit == false
247
- commit consumer.partition, consumer.offset
246
+ commit @current_consumer.partition, @current_consumer.offset
248
247
  end
249
248
 
250
249
  true
250
+ rescue StandardError
251
+ unlock
251
252
  end
252
253
 
253
254
  # Convenience method to fetch messages from the broker.
@@ -374,7 +375,12 @@ class Poseidon::ConsumerGroup
374
375
  # * let POS be our index position in CG and let N = size(PT)/size(CG)
375
376
  # * assign partitions from POS*N to (POS+1)*N-1
376
377
  def rebalance!
377
- @mutex.synchronize do
378
+ return if @rebalancing
379
+
380
+ @rebalancing = true
381
+ synchronize do
382
+ @rebalancing = nil
383
+
378
384
  release_all!
379
385
  reload_metadata
380
386
 
@@ -385,6 +391,11 @@ class Poseidon::ConsumerGroup
385
391
  return if responsible_for.nil?
386
392
 
387
393
  partition_list[responsible_for].each do |partition|
394
+ if @rebalancing
395
+ release_all!
396
+ break
397
+ end
398
+
388
399
  if consumer = claim!(partition.id)
389
400
  @consumers.push(consumer)
390
401
  end
@@ -401,6 +412,23 @@ class Poseidon::ConsumerGroup
401
412
  end
402
413
 
403
414
  private
415
+ def lock
416
+ @mutex.lock
417
+ end
418
+
419
+ def unlock(offset=nil)
420
+ raise "Mutex should be locked, possibly committing out of order" unless @mutex.locked?
421
+
422
+ if offset
423
+ @mutex.unlock if @current_consumer.offset == offset
424
+ else
425
+ @mutex.unlock
426
+ end
427
+ end
428
+
429
+ def synchronize
430
+ @mutex.synchronize { yield }
431
+ end
404
432
 
405
433
  # Claim the ownership of the partition for this consumer
406
434
  def claim!(partition)
@@ -409,6 +437,8 @@ class Poseidon::ConsumerGroup
409
437
  created = nil
410
438
 
411
439
  num_tries.times do
440
+ return nil if @rebalancing
441
+
412
442
  created = zk.create(claim_path(partition), id, {:ephemeral => true, :ignore => :node_exists})
413
443
  break if created
414
444
 
data/scenario/scenario.rb CHANGED
@@ -84,7 +84,7 @@ module Scenario
84
84
  sh "cd #{ROOT} && curl http://www.mirrorservice.org/sites/ftp.apache.org/kafka/#{VERSION}/kafka_2.10-#{VERSION}.tgz | tar xz"
85
85
  end
86
86
 
87
- def checkpoint!(timeout = 10)
87
+ def checkpoint!(timeout = 30)
88
88
  puts "--> Verifying #{@@total}"
89
89
  timeout.times do
90
90
  if numlines > @@total
@@ -29,7 +29,7 @@ describe Poseidon::ConsumerGroup do
29
29
  end
30
30
 
31
31
  let :zk_client do
32
- double "ZK", mkdir_p: nil, get: nil, set: nil, delete: nil, create: "/path", register: nil, children: ["my-group-UNIQUEID"], close: nil
32
+ double "ZK", mkdir_p: nil, get: nil, set: nil, delete: nil, create: "/path", register: nil, close: nil
33
33
  end
34
34
 
35
35
  let(:group) { described_class.new "my-group", ["localhost:29092", "localhost:29091"], ["localhost:22181"], "mytopic" }
@@ -37,6 +37,7 @@ describe Poseidon::ConsumerGroup do
37
37
 
38
38
  before do
39
39
  allow(ZK).to receive_messages(new: zk_client)
40
+ allow(zk_client).to receive(:children).and_return(nil, ["my-group-UNIQUEID"])
40
41
  allow(Poseidon::Cluster).to receive_messages(guid: "UNIQUEID")
41
42
  allow_any_instance_of(Poseidon::ConsumerGroup).to receive(:sleep)
42
43
  allow_any_instance_of(Poseidon::PartitionConsumer).to receive_messages(resolve_offset_if_necessary: 0)
@@ -170,6 +171,30 @@ describe Poseidon::ConsumerGroup do
170
171
  -> { subject.send :rebalance! }.should change { subject.claimed }.to([0])
171
172
  end
172
173
 
174
+ it "should wait for consumer to commit manually all messages before rebalancing" do
175
+ consumer = nil
176
+ subject.checkout(commit: false) { |c| consumer = c }
177
+
178
+ t = Thread.new { subject.send :rebalance! }
179
+
180
+ sleep 0.1
181
+ t.alive?.should == true
182
+
183
+ payloads = consumer.fetch
184
+ subject.commit(consumer.partition, payloads.shift.offset + 1)
185
+
186
+ sleep 0.1
187
+ t.alive?.should == true
188
+
189
+ payloads.each do |payload|
190
+ subject.commit(consumer.partition, payload.offset + 1)
191
+ end
192
+
193
+ sleep 0.1
194
+ t.alive?.should == false
195
+
196
+ t.join
197
+ end
173
198
  end
174
199
 
175
200
  describe "fetch" do
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: promiscuous-poseidon_cluster
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.1
4
+ version: 0.4.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Black Square Media
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2015-04-21 00:00:00.000000000 Z
12
+ date: 2015-05-07 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: poseidon