wooga-kafka-rb 0.0.7 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +12 -9
- data/Rakefile +11 -11
- data/lib/kafka/consumer.rb +4 -3
- data/spec/consumer_spec.rb +3 -3
- metadata +4 -4
data/README.md
CHANGED
@@ -1,13 +1,19 @@
|
|
1
1
|
# kafka-rb
|
2
|
-
kafka-rb allows you to produce messages to the Kafka distributed
|
2
|
+
kafka-rb allows you to produce and consume messages to / from the Kafka distributed messaging service.
|
3
|
+
This is an improved version of the original Ruby client written by Alexandro Crosa,
|
4
|
+
and is used in production at wooga.
|
3
5
|
|
4
6
|
## Requirements
|
5
|
-
You need to have access to your Kafka instance and be able to connect through TCP.
|
7
|
+
You need to have access to your Kafka instance and be able to connect through TCP.
|
8
|
+
You can obtain a copy and instructions on how to setup kafka at http://incubator.apache.org/kafka/
|
9
|
+
|
6
10
|
|
7
11
|
## Installation
|
8
|
-
sudo gem install kafka-rb
|
9
12
|
|
10
|
-
|
13
|
+
sudo gem install wooga-kafka-rb
|
14
|
+
|
15
|
+
(should work fine with JRuby, Ruby 1.8 and 1.9)
|
16
|
+
|
11
17
|
|
12
18
|
## Usage
|
13
19
|
|
@@ -55,8 +61,5 @@ sudo gem install kafka-rb
|
|
55
61
|
end
|
56
62
|
|
57
63
|
|
58
|
-
|
59
|
-
|
60
|
-
alejandrocrosa at(@) gmail.com
|
61
|
-
|
62
|
-
http://twitter.com/alejandrocrosa
|
64
|
+
## Questions?
|
65
|
+
tim.lossen@wooga.net
|
data/Rakefile
CHANGED
@@ -21,14 +21,13 @@ require 'rspec/core/rake_task'
|
|
21
21
|
|
22
22
|
spec = Gem::Specification.new do |s|
|
23
23
|
s.name = %q{wooga-kafka-rb}
|
24
|
-
s.version = "0.0.
|
24
|
+
s.version = "0.0.8"
|
25
25
|
|
26
26
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
27
27
|
s.authors = ["Alejandro Crosa", "Stefan Mees", "Tim Lossen"]
|
28
28
|
s.autorequire = %q{kafka-rb}
|
29
|
-
s.date = %
|
29
|
+
s.date = Time.now.strftime("%Y-%m-%d")
|
30
30
|
s.description = %q{kafka-rb allows you to produce and consume messages using the Kafka distributed publish/subscribe messaging service.}
|
31
|
-
s.email = %q{alejandrocrosa@gmail.com}
|
32
31
|
s.extra_rdoc_files = ["LICENSE"]
|
33
32
|
s.files = ["LICENSE", "README.md", "Rakefile", "lib/kafka", "lib/kafka/batch.rb", "lib/kafka/consumer.rb", "lib/kafka/io.rb", "lib/kafka/message.rb", "lib/kafka/producer.rb", "lib/kafka/request_type.rb", "lib/kafka/error_codes.rb", "lib/kafka.rb", "spec/batch_spec.rb", "spec/consumer_spec.rb", "spec/io_spec.rb", "spec/kafka_spec.rb", "spec/message_spec.rb", "spec/producer_spec.rb", "spec/spec_helper.rb"]
|
34
33
|
s.homepage = %q{http://github.com/wooga/kafka-rb}
|
@@ -54,14 +53,6 @@ Rake::GemPackageTask.new(spec) do |pkg|
|
|
54
53
|
pkg.gem_spec = spec
|
55
54
|
end
|
56
55
|
|
57
|
-
task :default => :rspec
|
58
|
-
|
59
|
-
desc "Run specs"
|
60
|
-
RSpec::Core::RakeTask.new do |t|
|
61
|
-
t.pattern = FileList['spec/**/*_spec.rb']
|
62
|
-
t.rspec_opts = %w(-fs --color)
|
63
|
-
end
|
64
|
-
|
65
56
|
desc "install the gem locally"
|
66
57
|
task :install => [:package] do
|
67
58
|
sh %{sudo gem install pkg/#{GEM}-#{GEM_VERSION}}
|
@@ -72,3 +63,12 @@ RSpec::Core::RakeTask.new(:rcov) do |t|
|
|
72
63
|
t.pattern = FileList['spec/**/*_spec.rb']
|
73
64
|
t.rcov = true
|
74
65
|
end
|
66
|
+
|
67
|
+
desc "Run specs"
|
68
|
+
RSpec::Core::RakeTask.new do |t|
|
69
|
+
t.pattern = FileList['spec/**/*_spec.rb']
|
70
|
+
t.rspec_opts = %w(-fs --color)
|
71
|
+
end
|
72
|
+
|
73
|
+
task :default => :spec
|
74
|
+
|
data/lib/kafka/consumer.rb
CHANGED
@@ -21,6 +21,7 @@ module Kafka
|
|
21
21
|
MAX_SIZE = 1024 * 1024 # 1 megabyte
|
22
22
|
DEFAULT_POLLING_INTERVAL = 2 # 2 seconds
|
23
23
|
MAX_OFFSETS = 1
|
24
|
+
LATEST_OFFSET = -1
|
24
25
|
EARLIEST_OFFSET = -2
|
25
26
|
|
26
27
|
attr_accessor :topic, :partition, :offset, :max_size, :request_type, :polling
|
@@ -46,7 +47,7 @@ module Kafka
|
|
46
47
|
end
|
47
48
|
|
48
49
|
def consume
|
49
|
-
self.offset ||=
|
50
|
+
self.offset ||= fetch_latest_offset
|
50
51
|
send_consume_request
|
51
52
|
data = read_data_response
|
52
53
|
parse_message_set_from(data)
|
@@ -54,14 +55,14 @@ module Kafka
|
|
54
55
|
nil
|
55
56
|
end
|
56
57
|
|
57
|
-
def
|
58
|
+
def fetch_latest_offset
|
58
59
|
send_offsets_request
|
59
60
|
read_offsets_response
|
60
61
|
end
|
61
62
|
|
62
63
|
def send_offsets_request
|
63
64
|
write(encoded_request_size)
|
64
|
-
write(encode_request(Kafka::RequestType::OFFSETS, topic, partition,
|
65
|
+
write(encode_request(Kafka::RequestType::OFFSETS, topic, partition, LATEST_OFFSET, MAX_OFFSETS))
|
65
66
|
end
|
66
67
|
|
67
68
|
def read_offsets_response
|
data/spec/consumer_spec.rb
CHANGED
@@ -157,7 +157,7 @@ describe Consumer do
|
|
157
157
|
|
158
158
|
it "should fetch initial offset if no offset is given" do
|
159
159
|
@consumer = Consumer.new
|
160
|
-
@consumer.should_receive(:
|
160
|
+
@consumer.should_receive(:fetch_latest_offset).exactly(:once).and_return(1000)
|
161
161
|
@consumer.should_receive(:send_consume_request).and_return(true)
|
162
162
|
@consumer.should_receive(:read_data_response).and_return("")
|
163
163
|
@consumer.consume
|
@@ -165,8 +165,8 @@ describe Consumer do
|
|
165
165
|
end
|
166
166
|
|
167
167
|
it "should encode an offset request" do
|
168
|
-
bytes = [Kafka::RequestType::OFFSETS].pack("n") + ["test".length].pack("n") + "test" + [0].pack("N") + [-
|
169
|
-
@consumer.encode_request(Kafka::RequestType::OFFSETS, "test", 0, -
|
168
|
+
bytes = [Kafka::RequestType::OFFSETS].pack("n") + ["test".length].pack("n") + "test" + [0].pack("N") + [-1].pack("q").reverse + [Kafka::Consumer::MAX_OFFSETS].pack("N")
|
169
|
+
@consumer.encode_request(Kafka::RequestType::OFFSETS, "test", 0, -1, Kafka::Consumer::MAX_OFFSETS).should eql(bytes)
|
170
170
|
end
|
171
171
|
|
172
172
|
it "should parse an offsets response" do
|
metadata
CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
5
5
|
segments:
|
6
6
|
- 0
|
7
7
|
- 0
|
8
|
-
-
|
9
|
-
version: 0.0.
|
8
|
+
- 8
|
9
|
+
version: 0.0.8
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Alejandro Crosa
|
@@ -16,7 +16,7 @@ autorequire: kafka-rb
|
|
16
16
|
bindir: bin
|
17
17
|
cert_chain: []
|
18
18
|
|
19
|
-
date: 2011-
|
19
|
+
date: 2011-12-12 00:00:00 +01:00
|
20
20
|
default_executable:
|
21
21
|
dependencies:
|
22
22
|
- !ruby/object:Gem::Dependency
|
@@ -33,7 +33,7 @@ dependencies:
|
|
33
33
|
type: :development
|
34
34
|
version_requirements: *id001
|
35
35
|
description: kafka-rb allows you to produce and consume messages using the Kafka distributed publish/subscribe messaging service.
|
36
|
-
email:
|
36
|
+
email:
|
37
37
|
executables: []
|
38
38
|
|
39
39
|
extensions: []
|