codeclimate-poseidon 0.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +21 -0
  3. data/.rspec +2 -0
  4. data/.travis.yml +14 -0
  5. data/.yardopts +8 -0
  6. data/CHANGES.md +31 -0
  7. data/Gemfile +13 -0
  8. data/LICENSE.txt +22 -0
  9. data/README.md +72 -0
  10. data/Rakefile +20 -0
  11. data/TODO.md +27 -0
  12. data/examples/consumer.rb +18 -0
  13. data/examples/producer.rb +9 -0
  14. data/lib/poseidon.rb +120 -0
  15. data/lib/poseidon/broker_pool.rb +86 -0
  16. data/lib/poseidon/cluster_metadata.rb +94 -0
  17. data/lib/poseidon/compressed_value.rb +23 -0
  18. data/lib/poseidon/compression.rb +30 -0
  19. data/lib/poseidon/compression/gzip_codec.rb +23 -0
  20. data/lib/poseidon/compression/snappy_codec.rb +29 -0
  21. data/lib/poseidon/connection.rb +169 -0
  22. data/lib/poseidon/fetched_message.rb +37 -0
  23. data/lib/poseidon/message.rb +151 -0
  24. data/lib/poseidon/message_conductor.rb +86 -0
  25. data/lib/poseidon/message_set.rb +80 -0
  26. data/lib/poseidon/message_to_send.rb +33 -0
  27. data/lib/poseidon/messages_for_broker.rb +56 -0
  28. data/lib/poseidon/messages_to_send.rb +47 -0
  29. data/lib/poseidon/messages_to_send_batch.rb +27 -0
  30. data/lib/poseidon/partition_consumer.rb +225 -0
  31. data/lib/poseidon/producer.rb +199 -0
  32. data/lib/poseidon/producer_compression_config.rb +37 -0
  33. data/lib/poseidon/protocol.rb +122 -0
  34. data/lib/poseidon/protocol/protocol_struct.rb +256 -0
  35. data/lib/poseidon/protocol/request_buffer.rb +77 -0
  36. data/lib/poseidon/protocol/response_buffer.rb +72 -0
  37. data/lib/poseidon/sync_producer.rb +161 -0
  38. data/lib/poseidon/topic_metadata.rb +89 -0
  39. data/lib/poseidon/version.rb +4 -0
  40. data/log/.gitkeep +0 -0
  41. data/poseidon.gemspec +27 -0
  42. data/spec/integration/multiple_brokers/consumer_spec.rb +45 -0
  43. data/spec/integration/multiple_brokers/metadata_failures_spec.rb +144 -0
  44. data/spec/integration/multiple_brokers/rebalance_spec.rb +69 -0
  45. data/spec/integration/multiple_brokers/round_robin_spec.rb +41 -0
  46. data/spec/integration/multiple_brokers/spec_helper.rb +60 -0
  47. data/spec/integration/simple/compression_spec.rb +23 -0
  48. data/spec/integration/simple/connection_spec.rb +35 -0
  49. data/spec/integration/simple/multiple_brokers_spec.rb +10 -0
  50. data/spec/integration/simple/simple_producer_and_consumer_spec.rb +121 -0
  51. data/spec/integration/simple/spec_helper.rb +16 -0
  52. data/spec/integration/simple/truncated_messages_spec.rb +46 -0
  53. data/spec/integration/simple/unavailable_broker_spec.rb +72 -0
  54. data/spec/spec_helper.rb +32 -0
  55. data/spec/test_cluster.rb +211 -0
  56. data/spec/unit/broker_pool_spec.rb +98 -0
  57. data/spec/unit/cluster_metadata_spec.rb +46 -0
  58. data/spec/unit/compression/gzip_codec_spec.rb +34 -0
  59. data/spec/unit/compression/snappy_codec_spec.rb +49 -0
  60. data/spec/unit/compression_spec.rb +17 -0
  61. data/spec/unit/connection_spec.rb +4 -0
  62. data/spec/unit/fetched_message_spec.rb +11 -0
  63. data/spec/unit/message_conductor_spec.rb +164 -0
  64. data/spec/unit/message_set_spec.rb +42 -0
  65. data/spec/unit/message_spec.rb +129 -0
  66. data/spec/unit/message_to_send_spec.rb +10 -0
  67. data/spec/unit/messages_for_broker_spec.rb +54 -0
  68. data/spec/unit/messages_to_send_batch_spec.rb +25 -0
  69. data/spec/unit/messages_to_send_spec.rb +63 -0
  70. data/spec/unit/partition_consumer_spec.rb +142 -0
  71. data/spec/unit/producer_compression_config_spec.rb +42 -0
  72. data/spec/unit/producer_spec.rb +51 -0
  73. data/spec/unit/protocol/request_buffer_spec.rb +16 -0
  74. data/spec/unit/protocol_spec.rb +54 -0
  75. data/spec/unit/sync_producer_spec.rb +156 -0
  76. data/spec/unit/topic_metadata_spec.rb +43 -0
  77. metadata +225 -0
@@ -0,0 +1,10 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe MessageToSend do
4
+ it "provides access to topic,value,key" do
5
+ mts = MessageToSend.new("hello_topic", "Hello World", "key")
6
+ expect(mts.topic).to eq("hello_topic")
7
+ expect(mts.value).to eq("Hello World")
8
+ expect(mts.key).to eq("key")
9
+ end
10
+ end
@@ -0,0 +1,54 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe MessagesForBroker do
4
+ context "twos message one to broker 0, partition 0, another to partition 1" do
5
+ before(:each) do
6
+ @messages = [ Message.new(:topic => "topic1",:value => "hi0"),
7
+ Message.new(:topic => "topic1",:value => "hi1")]
8
+
9
+ @compression_config = double('compression_config',
10
+ :compression_codec_for_topic => nil)
11
+
12
+ @mfb = MessagesForBroker.new(0)
13
+ @mfb.add(@messages[0], 0)
14
+ @mfb.add(@messages[1], 1)
15
+ end
16
+
17
+ it "provides the messages" do
18
+ expect(@mfb.messages.to_set).to eq(@messages.to_set)
19
+ end
20
+
21
+ it "is has a broker_id of 0" do
22
+ expect(@mfb.broker_id).to eq(0)
23
+ end
24
+
25
+ it "builds the protocol object correctly" do
26
+ protocol_object = @mfb.build_protocol_objects(@compression_config)
27
+
28
+ messages_for_topics = [
29
+ MessagesForTopic.new("topic1",
30
+ [
31
+ MessagesForPartition.new(0, MessageSet.new([@messages[0]])),
32
+ MessagesForPartition.new(1, MessageSet.new([@messages[1]])),
33
+ ])
34
+ ]
35
+ expect(protocol_object).to eq(messages_for_topics)
36
+ end
37
+
38
+ context "and topic is compressed" do
39
+ it "builds the protocol object correctly" do
40
+ allow(@compression_config).to receive_messages(:compression_codec_for_topic => Compression::GzipCodec)
41
+ protocol_object = @mfb.build_protocol_objects(@compression_config)
42
+
43
+ messages_for_topics = [
44
+ MessagesForTopic.new("topic1",
45
+ [
46
+ MessagesForPartition.new(0, MessageSet.new([@messages[0]]).compress(Compression::GzipCodec)),
47
+ MessagesForPartition.new(1, MessageSet.new([@messages[1]]).compress(Compression::GzipCodec)),
48
+ ])
49
+ ]
50
+ expect(protocol_object).to eq(messages_for_topics)
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,25 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe MessagesToSendBatch do
4
+ context "messages sent to two different brokers" do
5
+ before(:each) do
6
+ message_conductor = double('message_conductor')
7
+ allow(message_conductor).to receive(:destination).and_return([0,0],[1,1])
8
+
9
+ @messages = [
10
+ Message.new(:topic => "topic1", :value => "hi"),
11
+ Message.new(:topic => "topic1", :value => "hi")
12
+ ]
13
+ @batch = MessagesToSendBatch.new(@messages, message_conductor)
14
+ end
15
+
16
+ it "returns a couple messages brokers" do
17
+ expect(@batch.messages_for_brokers.size).to eq(2)
18
+ end
19
+
20
+ it "has all messages in the returned message brokers" do
21
+ messages = @batch.messages_for_brokers.map(&:messages).flatten
22
+ expect(messages.to_set).to eq(@messages.to_set)
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,63 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe MessagesToSend do
4
+ before(:each) do
5
+ @messages = []
6
+ @messages << Message.new(:topic => "test1", :value => "hi")
7
+ @messages << Message.new(:topic => "test2", :value => "hi")
8
+ @messages << Message.new(:topic => "test2", :value => "hi")
9
+
10
+
11
+ @cluster_metadata = double('cluster_metdata').as_null_object
12
+ @mts = MessagesToSend.new(@messages, @cluster_metadata)
13
+ end
14
+
15
+ describe "needing metadata" do
16
+ it "returns set of topics" do
17
+ expect(@mts.topic_set).to eq(Set.new(["test1","test2"]))
18
+ end
19
+
20
+ it "asks ClusterMetadata about having metadata" do
21
+ allow(@cluster_metadata).to receive(:have_metadata_for_topics?).and_return(true)
22
+
23
+ expect(@mts.needs_metadata?).to eq(false)
24
+ end
25
+ end
26
+
27
+ describe "sending" do
28
+ before(:each) do
29
+ @mfb = double('mfb', :messages => @messages)
30
+ @messages_for_brokers = [@mfb]
31
+
32
+ @mtsb = double('messages_to_send_batch').as_null_object
33
+ allow(@mtsb).to receive(:messages_for_brokers).and_return(@messages_for_brokers)
34
+
35
+ allow(MessagesToSendBatch).to receive(:new).and_return(@mtsb)
36
+ end
37
+
38
+ context "is successful" do
39
+ before(:each) do
40
+ @mts.messages_for_brokers(nil).each do |mfb|
41
+ @mts.successfully_sent(mfb.messages)
42
+ end
43
+ end
44
+
45
+ it "successfully sends all" do
46
+ expect(@mts.pending_messages?).to eq(false)
47
+ end
48
+ end
49
+
50
+ context "is not successful" do
51
+ before(:each) do
52
+ @mts.messages_for_brokers(nil).each do |mfb|
53
+ end
54
+ end
55
+
56
+ it "does not send all" do
57
+ @mts.messages_for_brokers(nil).each do |mfb|
58
+ end
59
+ expect(@mts.pending_messages?).to eq(true)
60
+ end
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,142 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe PartitionConsumer do
4
+ before(:each) do
5
+ @connection = double('connection')
6
+ allow(Connection).to receive(:new).and_return(@connection)
7
+
8
+ offset = Protocol::Offset.new(100)
9
+ partition_offsets = [Protocol::PartitionOffset.new(0, 0, [offset])]
10
+ @offset_response = [Protocol::TopicOffsetResponse.new("test_topic", partition_offsets)]
11
+ allow(@connection).to receive(:offset).and_return(@offset_response)
12
+ end
13
+
14
+ describe "creation" do
15
+ context "when passed unknown options" do
16
+ it "raises an ArgumentError" do
17
+ expect { PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0, :earliest_offset, :unknown => true) }.to raise_error(ArgumentError)
18
+ end
19
+ end
20
+
21
+ context "when passed an unknown offset" do
22
+ it "raises an ArgumentError" do
23
+ expect { PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0, :coolest_offset) }.to raise_error(ArgumentError)
24
+ end
25
+ end
26
+ end
27
+
28
+ describe "next offset" do
29
+ context "when offset is not set" do
30
+ it "resolves offset if it's not set" do
31
+ expect(@connection).to receive(:offset).and_return(@offset_response)
32
+ pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
33
+ 0, :earliest_offset)
34
+
35
+ pc.next_offset
36
+ end
37
+
38
+ it "returns resolved offset" do
39
+ pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
40
+ 0, :earliest_offset)
41
+ expect(pc.next_offset).to eq(100)
42
+ end
43
+ end
44
+
45
+ context "when offset is set" do
46
+ it "does not resolve it" do
47
+ pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
48
+ 0, 200)
49
+ pc.next_offset
50
+ end
51
+ end
52
+
53
+ context "when call returns an error" do
54
+ it "is raised" do
55
+ allow(@offset_response.first.partition_offsets.first).to receive(:error).and_return(2)
56
+ pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
57
+ 0, :earliest_offset)
58
+
59
+ expect { pc.next_offset }.to raise_error(Errors::InvalidMessage)
60
+ end
61
+ end
62
+
63
+ context "when no offset exists" do
64
+ it "sets offset to 0" do
65
+ pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
66
+ 0, :earliest_offset)
67
+
68
+ allow(@offset_response.first.partition_offsets.first).to receive(:offsets).and_return([])
69
+ expect(pc.next_offset).to eq(0)
70
+ end
71
+ end
72
+
73
+ context "when offset negative" do
74
+ it "resolves offset to one " do
75
+ pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
76
+ 0, -10)
77
+ expect(pc.next_offset).to eq(90)
78
+ end
79
+ end
80
+ end
81
+
82
+ describe "fetching messages" do
83
+ before(:each) do
84
+ message_set = MessageSet.new
85
+ message_set << Message.new(:value => "value", :key => "key", :offset => 90)
86
+ partition_fetch_response = Protocol::PartitionFetchResponse.new(0, 0, 100, message_set)
87
+ topic_fetch_response = Protocol::TopicFetchResponse.new('test_topic',
88
+ [partition_fetch_response])
89
+ @response = Protocol::FetchResponse.new(double('common'), [topic_fetch_response])
90
+
91
+ allow(@connection).to receive(:fetch).and_return(@response)
92
+ @pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0, :earliest_offset)
93
+ end
94
+
95
+ it "returns FetchedMessage objects" do
96
+ expect(@pc.fetch.first.class).to eq(FetchedMessage)
97
+ end
98
+
99
+ it "uses object defaults" do
100
+ expect(@connection).to receive(:fetch).with(10_000, 1, anything)
101
+ @pc.fetch
102
+ end
103
+
104
+ context "when options are passed" do
105
+ it "overrides object defaults" do
106
+ expect(@connection).to receive(:fetch).with(20_000, 1, anything)
107
+ @pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0, :earliest_offset, :max_wait_ms => 20_000)
108
+
109
+ @pc.fetch
110
+ end
111
+ end
112
+
113
+ context "when negative offset beyond beginning of partition is passed" do
114
+ it "starts from the earliest offset" do
115
+ @pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0, -10000)
116
+ pfr = @response.topic_fetch_responses.first.partition_fetch_responses.first
117
+ allow(pfr).to receive(:error).and_return(1, 1, 0)
118
+
119
+ @pc.fetch
120
+ end
121
+ end
122
+
123
+ context "when call returns an error" do
124
+ it "is raised" do
125
+ pfr = @response.topic_fetch_responses.first.partition_fetch_responses.first
126
+ allow(pfr).to receive(:error).and_return(2)
127
+
128
+ expect { @pc.fetch }.to raise_error(Errors::InvalidMessage)
129
+ end
130
+ end
131
+
132
+ it "sets the highwater mark" do
133
+ @pc.fetch
134
+ expect(@pc.highwater_mark).to eq(100)
135
+ end
136
+
137
+ it "sets the latest offset" do
138
+ @pc.fetch
139
+ expect(@pc.next_offset).to eq(91)
140
+ end
141
+ end
142
+ end
@@ -0,0 +1,42 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe ProducerCompressionConfig do
4
+ describe "creation" do
5
+ it "raises ArgumentError when codec is unknown" do
6
+ expect { ProducerCompressionConfig.new(:ripple, nil) }.to raise_error(ArgumentError)
7
+ end
8
+ end
9
+
10
+ context "no codec set" do
11
+ it "compresses no topics" do
12
+ pcc = ProducerCompressionConfig.new(nil,nil)
13
+ expect(pcc.compression_codec_for_topic("test")).to eq(false)
14
+ end
15
+ end
16
+
17
+ describe "none compression codec" do
18
+ it "compresses no topics" do
19
+ pcc = ProducerCompressionConfig.new(:none,nil)
20
+ expect(pcc.compression_codec_for_topic("test")).to eq(false)
21
+ end
22
+ end
23
+
24
+ describe "compression codec no topics specified" do
25
+ it "compresses any topic" do
26
+ pcc = ProducerCompressionConfig.new(:gzip,nil)
27
+ expect(pcc.compression_codec_for_topic("test")).to eq(Compression::GzipCodec)
28
+ end
29
+ end
30
+
31
+ describe "compression codec set, but only compress 'compressed' topic" do
32
+ it "compresses 'compressed' topic" do
33
+ pcc = ProducerCompressionConfig.new(:gzip, ["compressed"])
34
+ expect(pcc.compression_codec_for_topic("compressed")).to eq(Compression::GzipCodec)
35
+ end
36
+
37
+ it "does not compresses 'test' topic" do
38
+ pcc = ProducerCompressionConfig.new(:gzip, ["compressed"])
39
+ expect(pcc.compression_codec_for_topic("test")).to eq(false)
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,51 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe Producer do
4
+ it "requires brokers and client_id" do
5
+ expect { Producer.new }.to raise_error
6
+ end
7
+
8
+ it "raises ArgumentError on unknown arguments" do
9
+ expect { Producer.new([],"client_id", :unknown => true) }.to raise_error(ArgumentError)
10
+ end
11
+
12
+ it "raises ArgumentError unless brokers is an enumerable" do
13
+ expect { Producer.new("host:port","client_id") }.to raise_error(ArgumentError)
14
+ end
15
+
16
+ it "raises ProducerShutdown if we try to send to a shutdown producer" do
17
+ p = Producer.new(["host:port"],"client_id")
18
+ p.close
19
+ expect { p.send_messages([]) }.to raise_error(Errors::ProducerShutdownError)
20
+ end
21
+
22
+ it "accepts all options" do
23
+ expect { Producer.new([],"client_id", Producer::OPTION_DEFAULTS.dup) }.not_to raise_error
24
+ end
25
+
26
+ it "accepts socket_timeout_ms option" do
27
+ expect { Producer.new([],"client_id", socket_timeout_ms: 10_000) }.not_to raise_error
28
+ end
29
+
30
+ describe "sending messages" do
31
+ before(:each) do
32
+ @sync_producer = double('sync_producer').as_null_object
33
+ allow(SyncProducer).to receive(:new).and_return(@sync_producer)
34
+
35
+ @producer = Producer.new([], "client_id", :type => :sync)
36
+ end
37
+
38
+ it "turns MessagesToSend into Message objects" do
39
+ expect(@sync_producer).to receive(:send_messages).with(an_instance_of(Array)) do |array|
40
+ array.each { |obj| expect(obj).to be_an_instance_of(Message) }
41
+ end
42
+
43
+ m = MessageToSend.new("topic", "value")
44
+ @producer.send_messages([m])
45
+ end
46
+
47
+ it "raises an ArgumentError if you try to send a single message" do
48
+ expect { @producer.send_messages(MessageToSend.new("topic", "value")) }.to raise_error(ArgumentError)
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,16 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ require 'spec_helper'
4
+ include Protocol
5
+
6
+ RSpec.describe RequestBuffer do
7
+ subject(:buffer) { Poseidon::Protocol::RequestBuffer.new }
8
+
9
+ it 'appends UTF-8 strings' do
10
+ expect do
11
+ str = 'hello ümlaut'
12
+ buffer.append(str)
13
+ buffer.append(str.force_encoding(Encoding::BINARY))
14
+ end.to_not raise_error
15
+ end
16
+ end
@@ -0,0 +1,54 @@
1
+ require 'spec_helper'
2
+ include Protocol
3
+ RSpec.describe RequestCommon do
4
+ it "roundtrips" do
5
+ rc = RequestCommon.new(0,1,2,"client_id")
6
+
7
+ req_buffer = RequestBuffer.new
8
+ rc.write(req_buffer)
9
+
10
+ resp_buffer = ResponseBuffer.new(req_buffer.to_s)
11
+ rc_roundtrip = RequestCommon.read(resp_buffer)
12
+
13
+ expect(rc).to eq(rc_roundtrip)
14
+ end
15
+ end
16
+
17
+ RSpec.describe MetadataRequest do
18
+ it "roundtrips" do
19
+ rc = RequestCommon.new(0,1,2,"client_id")
20
+ mr = MetadataRequest.new(rc, ["topic1","topic2"])
21
+
22
+ req_buffer = RequestBuffer.new
23
+ mr.write(req_buffer)
24
+
25
+ resp_buffer = ResponseBuffer.new(req_buffer.to_s)
26
+ mr_roundtrip = MetadataRequest.read(resp_buffer)
27
+
28
+ expect(mr).to eq(mr_roundtrip)
29
+ end
30
+ end
31
+
32
+ RSpec.describe "objects with errors" do
33
+ it "returns objects that have errors" do
34
+ message_set = MessageSet.new
35
+ message_set << Message.new(:value => "value", :key => "key")
36
+ partition_fetch_response = PartitionFetchResponse.new(0, 5, 100, message_set)
37
+ topic_fetch_response = TopicFetchResponse.new('test_topic',
38
+ [partition_fetch_response])
39
+ response = FetchResponse.new(double('common'), [topic_fetch_response])
40
+
41
+ expect(response.objects_with_errors).to eq([partition_fetch_response])
42
+ end
43
+
44
+ it "raises error when asked" do
45
+ message_set = MessageSet.new
46
+ message_set << Message.new(:value => "value", :key => "key")
47
+ partition_fetch_response = PartitionFetchResponse.new(0, 5, 100, message_set)
48
+ topic_fetch_response = TopicFetchResponse.new('test_topic',
49
+ [partition_fetch_response])
50
+ response = FetchResponse.new(double('common'), [topic_fetch_response])
51
+
52
+ expect { response.raise_error_if_one_exists }.to raise_error
53
+ end
54
+ end