ralphrodkey-kafka-rb 0.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,20 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one or more
2
+ # contributor license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright ownership.
4
+ # The ASF licenses this file to You under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with
6
+ # the License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ require File.dirname(__FILE__) + '/spec_helper'
16
+
17
+ describe Kafka do
18
+ before(:each) do
19
+ end
20
+ end
@@ -0,0 +1,227 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one or more
2
+ # contributor license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright ownership.
4
+ # The ASF licenses this file to You under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with
6
+ # the License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ require File.dirname(__FILE__) + '/spec_helper'
16
+
17
+ describe Message do
18
+
19
+ def pack_v1_message bytes, attributes
20
+ [6 + bytes.length, 1, attributes, Zlib.crc32(bytes), bytes].pack "NCCNa*"
21
+ end
22
+
23
+ before(:each) do
24
+ @message = Message.new
25
+ end
26
+
27
+ describe "Kafka Message" do
28
+ it "should have a default magic number" do
29
+ Message::MAGIC_IDENTIFIER_DEFAULT.should eql(0)
30
+ end
31
+
32
+ it "should have a magic field, a checksum and a payload" do
33
+ [:magic, :checksum, :payload].each do |field|
34
+ @message.should respond_to(field.to_sym)
35
+ end
36
+ end
37
+
38
+ it "should set a default value of zero" do
39
+ @message.magic.should eql(Kafka::Message::MAGIC_IDENTIFIER_DEFAULT)
40
+ end
41
+
42
+ it "should allow to set a custom magic number" do
43
+ @message = Message.new("ale", 1)
44
+ @message.magic.should eql(1)
45
+ end
46
+
47
+ it "should have an empty payload by default" do
48
+ @message.payload.should == ""
49
+ end
50
+
51
+ it "should calculate the checksum (crc32 of a given message)" do
52
+ @message.payload = "ale"
53
+ @message.calculate_checksum.should eql(1120192889)
54
+ @message.payload = "alejandro"
55
+ @message.calculate_checksum.should eql(2865078607)
56
+ end
57
+
58
+ it "should say if the message is valid using the crc32 signature" do
59
+ @message.payload = "alejandro"
60
+ @message.checksum = 2865078607
61
+ @message.valid?.should eql(true)
62
+ @message.checksum = 0
63
+ @message.valid?.should eql(false)
64
+ @message = Message.new("alejandro", 0, 66666666) # 66666666 is a funny checksum
65
+ @message.valid?.should eql(false)
66
+ end
67
+ end
68
+
69
+ describe "parsing" do
70
+ it "should parse a version-0 message from bytes" do
71
+ bytes = [8, 0, 1120192889, 'ale'].pack('NCNa*')
72
+ message = Kafka::Message.parse_from(bytes).messages.first
73
+ message.valid?.should eql(true)
74
+ message.magic.should eql(0)
75
+ message.checksum.should eql(1120192889)
76
+ message.payload.should eql("ale")
77
+ end
78
+
79
+ it "should parse a version-1 message from bytes" do
80
+ bytes = [12, 1, 0, 755095536, 'martin'].pack('NCCNa*')
81
+ message = Kafka::Message.parse_from(bytes).messages.first
82
+ message.should be_valid
83
+ message.magic.should == 1
84
+ message.checksum.should == 755095536
85
+ message.payload.should == 'martin'
86
+ end
87
+
88
+ it "should raise an error if the magic number is not recognised" do
89
+ bytes = [12, 2, 0, 755095536, 'martin'].pack('NCCNa*') # 2 = some future format that's not yet invented
90
+ lambda {
91
+ Kafka::Message.parse_from(bytes)
92
+ }.should raise_error(RuntimeError, /Unsupported Kafka message version/)
93
+ end
94
+
95
+ it "should skip an incomplete message at the end of the response" do
96
+ bytes = [8, 0, 1120192889, 'ale'].pack('NCNa*')
97
+ bytes += [8].pack('N') # incomplete message (only length, rest is truncated)
98
+ message_set = Message.parse_from(bytes)
99
+ message_set.messages.size.should == 1
100
+ message_set.size.should == 12 # bytes consumed
101
+ end
102
+
103
+ it "should skip an incomplete message at the end of the response which has the same length as an empty message" do
104
+ bytes = [8, 0, 1120192889, 'ale'].pack('NCNa*')
105
+ bytes += [8, 0, 1120192889].pack('NCN') # incomplete message (payload is missing)
106
+ message_set = Message.parse_from(bytes)
107
+ message_set.messages.size.should == 1
108
+ message_set.size.should == 12 # bytes consumed
109
+ end
110
+
111
+ it "should read empty messages correctly" do
112
+ # empty message
113
+ bytes = [5, 0, 0, ''].pack('NCNa*')
114
+ messages = Message.parse_from(bytes).messages
115
+ messages.size.should == 1
116
+ messages.first.payload.should == ''
117
+ end
118
+
119
+ it "should parse a gzip-compressed message" do
120
+ compressed = 'H4sIAG0LI1AAA2NgYBBkZBB/9XN7YlJRYnJiCogCAH9lueQVAAAA'.unpack('m*').shift
121
+ bytes = [45, 1, 1, 1303540914, compressed].pack('NCCNa*')
122
+ message = Message.parse_from(bytes).messages.first
123
+ message.should be_valid
124
+ message.payload.should == 'abracadabra'
125
+ end
126
+
127
+ if Object.const_defined? "Snappy"
128
+ it "should parse a snappy-compressed message" do
129
+ cleartext = "abracadabra"
130
+ bytes = pack_v1_message cleartext, 0
131
+ compressed = Snappy.deflate(bytes)
132
+ bytes = pack_v1_message compressed, 2
133
+ message = Message.parse_from(bytes).messages.first
134
+ message.should be_valid
135
+ message.payload.should == cleartext
136
+ end
137
+
138
+ it "should recursively parse nested snappy compressed messages" do
139
+ uncompressed = pack_v1_message('abracadabra', 0)
140
+ uncompressed << pack_v1_message('foobar', 0)
141
+ compressed = pack_v1_message(Snappy.deflate(uncompressed), 2)
142
+ messages = Message.parse_from(compressed).messages
143
+ messages.map(&:payload).should == ['abracadabra', 'foobar']
144
+ messages.map(&:valid?).should == [true, true]
145
+ end
146
+
147
+ it "should support a mixture of snappy compressed and uncompressed messages" do
148
+ bytes = pack_v1_message(Snappy.deflate(pack_v1_message("compressed", 0)), 2)
149
+ bytes << pack_v1_message('uncompressed', 0)
150
+ messages = Message.parse_from(bytes).messages
151
+ messages.map(&:payload).should == ["compressed", "uncompressed"]
152
+ messages.map(&:valid?).should == [true, true]
153
+ end
154
+ end
155
+
156
+ it "should recursively parse nested gzip compressed messages" do
157
+ uncompressed = [17, 1, 0, 401275319, 'abracadabra'].pack('NCCNa*')
158
+ uncompressed << [12, 1, 0, 2666930069, 'foobar'].pack('NCCNa*')
159
+ compressed_io = StringIO.new('')
160
+ Zlib::GzipWriter.new(compressed_io).tap{|gzip| gzip << uncompressed; gzip.close }
161
+ compressed = compressed_io.string
162
+ bytes = [compressed.size + 6, 1, 1, Zlib.crc32(compressed), compressed].pack('NCCNa*')
163
+ messages = Message.parse_from(bytes).messages
164
+ messages.map(&:payload).should == ['abracadabra', 'foobar']
165
+ messages.map(&:valid?).should == [true, true]
166
+ end
167
+
168
+ it "should support a mixture of gzip compressed and uncompressed messages" do
169
+ compressed = 'H4sIAG0LI1AAA2NgYBBkZBB/9XN7YlJRYnJiCogCAH9lueQVAAAA'.unpack('m*').shift
170
+ bytes = [45, 1, 1, 1303540914, compressed].pack('NCCNa*')
171
+ bytes << [11, 1, 0, 907060870, 'hello'].pack('NCCNa*')
172
+ messages = Message.parse_from(bytes).messages
173
+ messages.map(&:payload).should == ['abracadabra', 'hello']
174
+ messages.map(&:valid?).should == [true, true]
175
+ end
176
+
177
+ it "should raise an error if the compression codec is not supported" do
178
+ bytes = [6, 1, 3, 0, ''].pack('NCCNa*') # 3 = some unknown future compression codec
179
+ lambda {
180
+ Kafka::Message.parse_from(bytes)
181
+ }.should raise_error(RuntimeError, /Unsupported Kafka compression codec/)
182
+ end
183
+ end
184
+
185
+ describe "#ensure_snappy!" do
186
+ let(:message) { Kafka::Message.new }
187
+ before { Kafka::Message.instance_variable_set :@snappy, nil }
188
+
189
+ subject { message.ensure_snappy! { 42 } }
190
+
191
+ if Object.const_defined? "Snappy"
192
+ context "when snappy is available" do
193
+ before { Object.stub! :const_defined? => true }
194
+ it { should == 42 }
195
+ end
196
+ end
197
+
198
+ context "when snappy is not available" do
199
+ before { Object.stub! :const_defined? => false }
200
+
201
+ it "raises an error" do
202
+ expect { message.ensure_snappy! { 42 } }.to raise_error
203
+ end
204
+ end
205
+ end
206
+
207
+ describe ".ensure_snappy!" do
208
+ before { Kafka::Message.instance_variable_set :@snappy, nil }
209
+
210
+ subject { Kafka::Message.ensure_snappy! { 42 } }
211
+
212
+ if Object.const_defined? "Snappy"
213
+ context "when snappy is available" do
214
+ before { Object.stub! :const_defined? => true }
215
+ it { should == 42 }
216
+ end
217
+ end
218
+
219
+ context "when snappy is not available" do
220
+ before { Object.stub! :const_defined? => false }
221
+
222
+ it "raises an error" do
223
+ expect { Kafka::Message.ensure_snappy! { 42 } }.to raise_error
224
+ end
225
+ end
226
+ end
227
+ end
@@ -0,0 +1,74 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one or more
2
+ # contributor license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright ownership.
4
+ # The ASF licenses this file to You under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with
6
+ # the License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ require File.dirname(__FILE__) + '/spec_helper'
17
+
18
+ describe MultiProducer do
19
+ before(:each) do
20
+ @mocked_socket = mock(TCPSocket)
21
+ TCPSocket.stub!(:new).and_return(@mocked_socket) # don't use a real socket
22
+ end
23
+
24
+ describe "Kafka Producer" do
25
+ it "should set default host and port if none is specified" do
26
+ subject.host.should eql("localhost")
27
+ subject.port.should eql(9092)
28
+ end
29
+
30
+ it "should have compression" do
31
+ subject.should respond_to :compression
32
+ described_class.new(:compression => Kafka::Message::SNAPPY_COMPRESSION).compression.should == Kafka::Message::SNAPPY_COMPRESSION
33
+ described_class.new.compression.should == Kafka::Message::NO_COMPRESSION
34
+ end
35
+
36
+ it "sends single messages" do
37
+ message = Kafka::Message.new("ale")
38
+ encoded = Kafka::Encoder.produce("test", 0, message)
39
+
40
+ subject.should_receive(:write).with(encoded).and_return(encoded.length)
41
+ subject.push("test", message, :partition => 0).should == encoded.length
42
+ end
43
+
44
+ it "sends multiple messages" do
45
+ messages = [Kafka::Message.new("ale"), Kafka::Message.new("beer")]
46
+ reqs = [
47
+ Kafka::ProducerRequest.new("topic", messages[0]),
48
+ Kafka::ProducerRequest.new("topic", messages[1]),
49
+ ]
50
+ encoded = Encoder.multiproduce(reqs)
51
+
52
+ subject.should_receive(:write).with(encoded).and_return(encoded.length)
53
+ subject.multi_push(reqs).should == encoded.length
54
+ end
55
+
56
+ it "should compress messages" do
57
+ subject.compression = Kafka::Message::SNAPPY_COMPRESSION
58
+ @mocked_socket.stub! :write => 0
59
+ messages = [Kafka::Message.new("ale"), Kafka::Message.new("beer")]
60
+
61
+ encoded = Encoder.produce("test", 0, messages[0])
62
+ Encoder.should_receive(:produce).with("test", 0, messages[0], subject.compression).and_return encoded
63
+ subject.push("test", messages[0], :partition => 0)
64
+
65
+ reqs = [
66
+ Kafka::ProducerRequest.new("topic", messages[0]),
67
+ Kafka::ProducerRequest.new("topic", messages[1]),
68
+ ]
69
+ encoded = Encoder.multiproduce(reqs)
70
+ Encoder.should_receive(:multiproduce).with(reqs, subject.compression)
71
+ subject.multi_push(reqs)
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,38 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one or more
2
+ # contributor license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright ownership.
4
+ # The ASF licenses this file to You under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with
6
+ # the License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ require File.dirname(__FILE__) + '/spec_helper'
17
+
18
+ describe ProducerRequest do
19
+ let(:message) { Kafka::Message.new }
20
+ let(:req) { described_class.new("topic", message) }
21
+
22
+ it "has a topic" do
23
+ req.topic = "topic"
24
+ end
25
+
26
+ it "has a set of messages" do
27
+ req.messages.should == [message]
28
+ end
29
+
30
+ it "has a default partition" do
31
+ req.partition.should == 0
32
+ end
33
+
34
+ it "can use a user-specified partition" do
35
+ req = described_class.new("topic", message, :partition => 42)
36
+ req.partition.should == 42
37
+ end
38
+ end
@@ -0,0 +1,71 @@
1
+ # encoding: utf-8
2
+
3
+ # Licensed to the Apache Software Foundation (ASF) under one or more
4
+ # contributor license agreements. See the NOTICE file distributed with
5
+ # this work for additional information regarding copyright ownership.
6
+ # The ASF licenses this file to You under the Apache License, Version 2.0
7
+ # (the "License"); you may not use this file except in compliance with
8
+ # the License. You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ require File.dirname(__FILE__) + '/spec_helper'
18
+
19
+ describe Producer do
20
+
21
+ before(:each) do
22
+ @mocked_socket = mock(TCPSocket)
23
+ TCPSocket.stub!(:new).and_return(@mocked_socket) # don't use a real socket
24
+ @producer = Producer.new
25
+ end
26
+
27
+ describe "Kafka Producer" do
28
+ it "should have a topic and a partition" do
29
+ @producer.should respond_to(:topic)
30
+ @producer.should respond_to(:partition)
31
+ end
32
+
33
+ it "should have compression" do
34
+ @producer.should respond_to :compression
35
+ Producer.new(:compression => 1).compression.should == 1
36
+ Producer.new.compression.should == 0
37
+ end
38
+
39
+ it "should set a topic and partition on initialize" do
40
+ @producer = Producer.new({ :host => "localhost", :port => 9092, :topic => "testing" })
41
+ @producer.topic.should eql("testing")
42
+ @producer.partition.should eql(0)
43
+ @producer = Producer.new({ :topic => "testing", :partition => 3 })
44
+ @producer.partition.should eql(3)
45
+ end
46
+
47
+ it "should set default host and port if none is specified" do
48
+ @producer = Producer.new
49
+ @producer.host.should eql("localhost")
50
+ @producer.port.should eql(9092)
51
+ end
52
+ end
53
+
54
+ it "should send messages" do
55
+ @producer.should_receive(:write).and_return(32)
56
+ message = Kafka::Message.new("ale")
57
+ @producer.push(message).should eql(32)
58
+ end
59
+
60
+ describe "Message Batching" do
61
+ it "should batch messages and send them at once" do
62
+ message1 = Kafka::Message.new("one")
63
+ message2 = Kafka::Message.new("two")
64
+ @producer.should_receive(:push).with([message1, message2]).exactly(:once).and_return(nil)
65
+ @producer.batch do |messages|
66
+ messages << message1
67
+ messages << message2
68
+ end
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,18 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one or more
2
+ # contributor license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright ownership.
4
+ # The ASF licenses this file to You under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with
6
+ # the License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ require 'rubygems'
16
+ require 'kafka'
17
+
18
+ include Kafka