kafka-rb 0.0.10 → 0.0.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Rakefile CHANGED
@@ -14,14 +14,14 @@
14
14
  # limitations under the License.
15
15
 
16
16
  require 'rubygems'
17
- require 'rake/gempackagetask'
17
+ require 'rubygems/package_task'
18
18
  require 'rubygems/specification'
19
19
  require 'date'
20
20
  require 'rspec/core/rake_task'
21
21
 
22
22
  spec = Gem::Specification.new do |s|
23
23
  s.name = %q{kafka-rb}
24
- s.version = "0.0.10"
24
+ s.version = "0.0.11"
25
25
 
26
26
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
27
27
  s.authors = ["Alejandro Crosa", "Stefan Mees", "Tim Lossen", "Liam Stewart"]
@@ -29,10 +29,10 @@ spec = Gem::Specification.new do |s|
29
29
  s.date = Time.now.strftime("%Y-%m-%d")
30
30
  s.description = %q{kafka-rb allows you to produce and consume messages using the Kafka distributed publish/subscribe messaging service.}
31
31
  s.extra_rdoc_files = ["LICENSE"]
32
- s.files = ["LICENSE", "README.md", "Rakefile", "lib/kafka", "lib/kafka/batch.rb", "lib/kafka/consumer.rb", "lib/kafka/io.rb", "lib/kafka/message.rb", "lib/kafka/producer.rb", "lib/kafka/request_type.rb", "lib/kafka/error_codes.rb", "lib/kafka.rb", "spec/batch_spec.rb", "spec/consumer_spec.rb", "spec/io_spec.rb", "spec/kafka_spec.rb", "spec/message_spec.rb", "spec/producer_spec.rb", "spec/spec_helper.rb"]
32
+ s.files = ["LICENSE", "README.md", "Rakefile"] + Dir.glob("lib/**/*.rb")
33
+ s.test_files = Dir.glob("spec/**/*.rb")
33
34
  s.homepage = %q{http://github.com/acrosa/kafka-rb}
34
35
  s.require_paths = ["lib"]
35
- s.rubygems_version = %q{1.3.7}
36
36
  s.summary = %q{A Ruby client for the Kafka distributed publish/subscribe messaging service}
37
37
 
38
38
  if s.respond_to? :specification_version then
@@ -0,0 +1,61 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one or more
2
+ # contributor license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright ownership.
4
+ # The ASF licenses this file to You under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with
6
+ # the License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ module Kafka
17
+ module Encoder
18
+ def self.message(message)
19
+ payload = \
20
+ if RUBY_VERSION[0,3] == "1.8" # Use old iconv on Ruby 1.8 for encoding
21
+ Iconv.new('UTF-8//IGNORE', 'UTF-8').iconv(message.payload.to_s)
22
+ else
23
+ message.payload.to_s.force_encoding(Encoding::ASCII_8BIT)
24
+ end
25
+ data = [message.magic].pack("C") + [message.calculate_checksum].pack("N") + payload
26
+
27
+ [data.length].pack("N") + data
28
+ end
29
+
30
+ def self.message_block(topic, partition, messages)
31
+ message_set = Array(messages).collect { |message|
32
+ self.message(message)
33
+ }.join("")
34
+
35
+ topic = [topic.length].pack("n") + topic
36
+ partition = [partition].pack("N")
37
+ messages = [message_set.length].pack("N") + message_set
38
+
39
+ return topic + partition + messages
40
+ end
41
+
42
+ def self.produce(topic, partition, messages)
43
+ request = [RequestType::PRODUCE].pack("n")
44
+ data = request + self.message_block(topic, partition, messages)
45
+
46
+ return [data.length].pack("N") + data
47
+ end
48
+
49
+ def self.multiproduce(producer_requests)
50
+ part_set = Array(producer_requests).map { |req|
51
+ self.message_block(req.topic, req.partition, req.messages)
52
+ }
53
+
54
+ request = [RequestType::MULTIPRODUCE].pack("n")
55
+ parts = [part_set.length].pack("n") + part_set.join("")
56
+ data = request + parts
57
+
58
+ return [data.length].pack("N") + data
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,34 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one or more
2
+ # contributor license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright ownership.
4
+ # The ASF licenses this file to You under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with
6
+ # the License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ module Kafka
16
+ class MultiProducer
17
+ include Kafka::IO
18
+
19
+ def initialize(options={})
20
+ self.host = options[:host] || HOST
21
+ self.port = options[:port] || PORT
22
+ self.connect(self.host, self.port)
23
+ end
24
+
25
+ def send(topic, messages, options={})
26
+ partition = options[:partition] || 0
27
+ self.write(Encoder.produce(topic, partition, messages))
28
+ end
29
+
30
+ def multi_send(producer_requests)
31
+ self.write(Encoder.multiproduce(producer_requests))
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,26 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one or more
2
+ # contributor license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright ownership.
4
+ # The ASF licenses this file to You under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with
6
+ # the License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ module Kafka
17
+ class ProducerRequest
18
+ attr_accessor :topic, :messages, :partition
19
+
20
+ def initialize(topic, messages, options={})
21
+ self.topic = topic
22
+ self.partition = options[:partition] || 0
23
+ self.messages = Array(messages)
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,173 @@
1
+ # encoding: utf-8
2
+
3
+ # Licensed to the Apache Software Foundation (ASF) under one or more
4
+ # contributor license agreements. See the NOTICE file distributed with
5
+ # this work for additional information regarding copyright ownership.
6
+ # The ASF licenses this file to You under the Apache License, Version 2.0
7
+ # (the "License"); you may not use this file except in compliance with
8
+ # the License. You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ require File.dirname(__FILE__) + '/spec_helper'
18
+
19
+ describe Encoder do
20
+ def check_message(bytes, message)
21
+ encoded = [message.magic].pack("C") + [message.calculate_checksum].pack("N") + message.payload
22
+ encoded = [encoded.length].pack("N") + encoded
23
+ bytes.should == encoded
24
+ end
25
+
26
+ describe "Message Encoding" do
27
+ it "should encode a message" do
28
+ message = Kafka::Message.new("alejandro")
29
+ check_message(described_class.message(message), message)
30
+ end
31
+
32
+ it "should encode an empty message" do
33
+ message = Kafka::Message.new
34
+ check_message(described_class.message(message), message)
35
+ end
36
+
37
+ it "should encode strings containing non-ASCII characters" do
38
+ message = Kafka::Message.new("ümlaut")
39
+ encoded = described_class.message(message)
40
+ message = Kafka::Message.parse_from(encoded).messages.first
41
+ if RUBY_VERSION[0,3] == "1.8" # Use old iconv on Ruby 1.8 for encoding
42
+ ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
43
+ ic.iconv(message.payload).should eql("ümlaut")
44
+ else
45
+ message.payload.force_encoding(Encoding::UTF_8).should eql("ümlaut")
46
+ end
47
+ end
48
+ end
49
+
50
+ describe "produce" do
51
+ it "should binary encode an empty request" do
52
+ bytes = described_class.produce("test", 0, [])
53
+ bytes.length.should eql(20)
54
+ bytes.should eql("\000\000\000\020\000\000\000\004test\000\000\000\000\000\000\000\000")
55
+ end
56
+
57
+ it "should binary encode a request with a message, using a specific wire format" do
58
+ message = Kafka::Message.new("ale")
59
+ bytes = described_class.produce("test", 3, message)
60
+ data_size = bytes[0, 4].unpack("N").shift
61
+ request_id = bytes[4, 2].unpack("n").shift
62
+ topic_length = bytes[6, 2].unpack("n").shift
63
+ topic = bytes[8, 4]
64
+ partition = bytes[12, 4].unpack("N").shift
65
+ messages_length = bytes[16, 4].unpack("N").shift
66
+ messages = bytes[20, messages_length]
67
+
68
+ bytes.length.should eql(32)
69
+ data_size.should eql(28)
70
+ request_id.should eql(0)
71
+ topic_length.should eql(4)
72
+ topic.should eql("test")
73
+ partition.should eql(3)
74
+ messages_length.should eql(12)
75
+ end
76
+ end
77
+
78
+ describe "multiproduce" do
79
+ it "encodes an empty request" do
80
+ bytes = described_class.multiproduce([])
81
+ bytes.length.should == 8
82
+ bytes.should == "\x00\x00\x00\x04\x00\x03\x00\x00"
83
+ end
84
+
85
+ it "encodes a request with a single topic/partition" do
86
+ message = Kafka::Message.new("ale")
87
+ bytes = described_class.multiproduce(Kafka::ProducerRequest.new("test", message))
88
+
89
+ req_length = bytes[0, 4].unpack("N").shift
90
+ req_type = bytes[4, 2].unpack("n").shift
91
+ tp_count = bytes[6, 2].unpack("n").shift
92
+
93
+ req_type.should == Kafka::RequestType::MULTIPRODUCE
94
+ tp_count.should == 1
95
+
96
+ topic_length = bytes[8, 2].unpack("n").shift
97
+ topic = bytes[10, 4]
98
+ partition = bytes[14, 4].unpack("N").shift
99
+ messages_length = bytes[18, 4].unpack("N").shift
100
+ messages_data = bytes[22, messages_length]
101
+
102
+ topic_length.should == 4
103
+ topic.should == "test"
104
+ partition.should == 0
105
+ messages_length.should == 12
106
+ check_message(messages_data, message)
107
+ end
108
+
109
+ it "encodes a request with a single topic/partition but multiple messages" do
110
+ messages = [Kafka::Message.new("ale"), Kafka::Message.new("beer")]
111
+ bytes = described_class.multiproduce(Kafka::ProducerRequest.new("test", messages))
112
+
113
+ req_length = bytes[0, 4].unpack("N").shift
114
+ req_type = bytes[4, 2].unpack("n").shift
115
+ tp_count = bytes[6, 2].unpack("n").shift
116
+
117
+ req_type.should == Kafka::RequestType::MULTIPRODUCE
118
+ tp_count.should == 1
119
+
120
+ topic_length = bytes[8, 2].unpack("n").shift
121
+ topic = bytes[10, 4]
122
+ partition = bytes[14, 4].unpack("N").shift
123
+ messages_length = bytes[18, 4].unpack("N").shift
124
+ messages_data = bytes[22, messages_length]
125
+
126
+ topic_length.should == 4
127
+ topic.should == "test"
128
+ partition.should == 0
129
+ messages_length.should == 25
130
+ check_message(messages_data[0, 12], messages[0])
131
+ check_message(messages_data[12, 13], messages[1])
132
+ end
133
+
134
+ it "encodes a request with multiple topic/partitions" do
135
+ messages = [Kafka::Message.new("ale"), Kafka::Message.new("beer")]
136
+ bytes = described_class.multiproduce([
137
+ Kafka::ProducerRequest.new("test", messages[0]),
138
+ Kafka::ProducerRequest.new("topic", messages[1], partition: 1),
139
+ ])
140
+
141
+ req_length = bytes[0, 4].unpack("N").shift
142
+ req_type = bytes[4, 2].unpack("n").shift
143
+ tp_count = bytes[6, 2].unpack("n").shift
144
+
145
+ req_type.should == Kafka::RequestType::MULTIPRODUCE
146
+ tp_count.should == 2
147
+
148
+ topic_length = bytes[8, 2].unpack("n").shift
149
+ topic = bytes[10, 4]
150
+ partition = bytes[14, 4].unpack("N").shift
151
+ messages_length = bytes[18, 4].unpack("N").shift
152
+ messages_data = bytes[22, 12]
153
+
154
+ topic_length.should == 4
155
+ topic.should == "test"
156
+ partition.should == 0
157
+ messages_length.should == 12
158
+ check_message(messages_data[0, 12], messages[0])
159
+
160
+ topic_length = bytes[34, 2].unpack("n").shift
161
+ topic = bytes[36, 5]
162
+ partition = bytes[41, 4].unpack("N").shift
163
+ messages_length = bytes[45, 4].unpack("N").shift
164
+ messages_data = bytes[49, 13]
165
+
166
+ topic_length.should == 5
167
+ topic.should == "topic"
168
+ partition.should == 1
169
+ messages_length.should == 13
170
+ check_message(messages_data[0, 13], messages[1])
171
+ end
172
+ end
173
+ end
@@ -0,0 +1,50 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one or more
2
+ # contributor license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright ownership.
4
+ # The ASF licenses this file to You under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with
6
+ # the License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ require File.dirname(__FILE__) + '/spec_helper'
17
+
18
+ describe MultiProducer do
19
+ before(:each) do
20
+ @mocked_socket = mock(TCPSocket)
21
+ TCPSocket.stub!(:new).and_return(@mocked_socket) # don't use a real socket
22
+ end
23
+
24
+ describe "Kafka Producer" do
25
+ it "should set default host and port if none is specified" do
26
+ subject.host.should eql("localhost")
27
+ subject.port.should eql(9092)
28
+ end
29
+
30
+ it "sends single messages" do
31
+ message = Kafka::Message.new("ale")
32
+ encoded = Kafka::Encoder.produce("test", 0, message)
33
+
34
+ subject.should_receive(:write).with(encoded).and_return(encoded.length)
35
+ subject.send("test", message, partition: 0).should == encoded.length
36
+ end
37
+
38
+ it "sends multiple messages" do
39
+ messages = [Kafka::Message.new("ale"), Kafka::Message.new("beer")]
40
+ reqs = [
41
+ Kafka::ProducerRequest.new("topic", messages[0]),
42
+ Kafka::ProducerRequest.new("topic", messages[1]),
43
+ ]
44
+ encoded = Encoder.multiproduce(reqs)
45
+
46
+ subject.should_receive(:write).with(encoded).and_return(encoded.length)
47
+ subject.multi_send(reqs).should == encoded.length
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,38 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one or more
2
+ # contributor license agreements. See the NOTICE file distributed with
3
+ # this work for additional information regarding copyright ownership.
4
+ # The ASF licenses this file to You under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with
6
+ # the License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ require File.dirname(__FILE__) + '/spec_helper'
17
+
18
+ describe ProducerRequest do
19
+ let(:message) { Kafka::Message.new }
20
+ let(:req) { described_class.new("topic", message) }
21
+
22
+ it "has a topic" do
23
+ req.topic = "topic"
24
+ end
25
+
26
+ it "has a set of messages" do
27
+ req.messages.should == [message]
28
+ end
29
+
30
+ it "has a default partition" do
31
+ req.partition.should == 0
32
+ end
33
+
34
+ it "can use a user-specified partition" do
35
+ req = described_class.new("topic", message, partition: 42)
36
+ req.partition.should == 42
37
+ end
38
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: kafka-rb
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.10
4
+ version: 0.0.11
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire: kafka-rb
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2012-09-11 00:00:00.000000000 Z
15
+ date: 2012-09-12 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: rspec
@@ -43,17 +43,23 @@ files:
43
43
  - Rakefile
44
44
  - lib/kafka/batch.rb
45
45
  - lib/kafka/consumer.rb
46
+ - lib/kafka/encoder.rb
47
+ - lib/kafka/error_codes.rb
46
48
  - lib/kafka/io.rb
47
49
  - lib/kafka/message.rb
50
+ - lib/kafka/multi_producer.rb
48
51
  - lib/kafka/producer.rb
52
+ - lib/kafka/producer_request.rb
49
53
  - lib/kafka/request_type.rb
50
- - lib/kafka/error_codes.rb
51
54
  - lib/kafka.rb
52
55
  - spec/batch_spec.rb
53
56
  - spec/consumer_spec.rb
57
+ - spec/encoder_spec.rb
54
58
  - spec/io_spec.rb
55
59
  - spec/kafka_spec.rb
56
60
  - spec/message_spec.rb
61
+ - spec/multi_producer_spec.rb
62
+ - spec/producer_request_spec.rb
57
63
  - spec/producer_spec.rb
58
64
  - spec/spec_helper.rb
59
65
  homepage: http://github.com/acrosa/kafka-rb
@@ -80,4 +86,14 @@ rubygems_version: 1.8.24
80
86
  signing_key:
81
87
  specification_version: 3
82
88
  summary: A Ruby client for the Kafka distributed publish/subscribe messaging service
83
- test_files: []
89
+ test_files:
90
+ - spec/batch_spec.rb
91
+ - spec/consumer_spec.rb
92
+ - spec/encoder_spec.rb
93
+ - spec/io_spec.rb
94
+ - spec/kafka_spec.rb
95
+ - spec/message_spec.rb
96
+ - spec/multi_producer_spec.rb
97
+ - spec/producer_request_spec.rb
98
+ - spec/producer_spec.rb
99
+ - spec/spec_helper.rb