ruby-kafka 0.1.0.pre.alpha → 0.1.0.pre.alpha2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +21 -6
- data/docker-compose.yml +35 -0
- data/lib/kafka.rb +13 -3
- data/lib/kafka/broker.rb +50 -0
- data/lib/kafka/broker_pool.rb +77 -0
- data/lib/kafka/client.rb +22 -0
- data/lib/kafka/connection.rb +70 -25
- data/lib/kafka/message.rb +12 -0
- data/lib/kafka/message_set.rb +24 -0
- data/lib/kafka/producer.rb +35 -15
- data/lib/kafka/protocol.rb +15 -0
- data/lib/kafka/protocol/decoder.rb +40 -2
- data/lib/kafka/protocol/encoder.rb +58 -0
- data/lib/kafka/protocol/metadata_response.rb +97 -6
- data/lib/kafka/protocol/produce_request.rb +1 -10
- data/lib/kafka/protocol/produce_response.rb +16 -10
- data/lib/kafka/version.rb +1 -1
- data/lib/ruby-kafka.rb +3 -0
- data/test-setup.sh +3 -0
- metadata +10 -3
- data/lib/kafka/cluster.rb +0 -53
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 945e87a87fcfebd2808de4203027613846a2f7ad
|
4
|
+
data.tar.gz: f43c576a56aea2f49ef1047aec56f376ec2be0b0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c51ccf72b3822a773d013c68fe77665d7d5e9b4021131cb97d8bed65e47c681be18d89f718b63840ef5c463991b0c56c042b3ab747753c23b7f88c0b6131d8d3
|
7
|
+
data.tar.gz: 4c5c41e3858562fcdac0540652f5d3d8f21334a841b717f7f6281f1ff15a58c3ba165f518fbce9778669d5d35279e5242bb3ca3d1afce62a5c40c01b790e1720
|
data/README.md
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
# Kafka
|
2
2
|
|
3
|
-
|
3
|
+
A Ruby client library for the Kafka distributed log system. The focus of this library will be operational simplicity, with good logging and metrics that can make debugging issues easier.
|
4
4
|
|
5
|
-
|
5
|
+
This library is still in pre-alpha stage, but development is ongoing. Current efforts are focused on implementing a solid Producer client. The next step will be implementing a client for the Kafka 0.9 Consumer API.
|
6
6
|
|
7
7
|
## Installation
|
8
8
|
|
9
9
|
Add this line to your application's Gemfile:
|
10
10
|
|
11
11
|
```ruby
|
12
|
-
gem 'kafka'
|
12
|
+
gem 'ruby-kafka'
|
13
13
|
```
|
14
14
|
|
15
15
|
And then execute:
|
@@ -18,11 +18,26 @@ And then execute:
|
|
18
18
|
|
19
19
|
Or install it yourself as:
|
20
20
|
|
21
|
-
$ gem install kafka
|
21
|
+
$ gem install ruby-kafka
|
22
22
|
|
23
23
|
## Usage
|
24
24
|
|
25
|
-
|
25
|
+
```ruby
|
26
|
+
kafka = Kafka.new(
|
27
|
+
seed_brokers: ["kafka1:9092", "kafka2:9092"],
|
28
|
+
client_id: "my-app",
|
29
|
+
logger: Logger.new($stderr),
|
30
|
+
)
|
31
|
+
|
32
|
+
producer = kafka.get_producer
|
33
|
+
|
34
|
+
# `write` will buffer the message in the producer.
|
35
|
+
producer.write("hello1", key: "x", topic: "test-messages", partition: 0)
|
36
|
+
producer.write("hello2", key: "y", topic: "test-messages", partition: 1)
|
37
|
+
|
38
|
+
# `flush` will send the buffered messages to the cluster.
|
39
|
+
producer.flush
|
40
|
+
```
|
26
41
|
|
27
42
|
## Development
|
28
43
|
|
@@ -32,7 +47,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
|
|
32
47
|
|
33
48
|
## Contributing
|
34
49
|
|
35
|
-
Bug reports and pull requests are welcome on GitHub at https://github.com/
|
50
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/zendesk/ruby-kafka.
|
36
51
|
|
37
52
|
|
38
53
|
## Copyright and license
|
data/docker-compose.yml
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
zookeeper:
|
2
|
+
image: wurstmeister/zookeeper
|
3
|
+
ports:
|
4
|
+
- "2181"
|
5
|
+
kafka1:
|
6
|
+
image: wurstmeister/kafka:0.8.2.0
|
7
|
+
ports:
|
8
|
+
- "9992:9092"
|
9
|
+
links:
|
10
|
+
- zookeeper:zk
|
11
|
+
environment:
|
12
|
+
KAFKA_ADVERTISED_HOST_NAME: "192.168.42.45"
|
13
|
+
KAFKA_CREATE_TOPICS: "test-messages:5:3"
|
14
|
+
volumes:
|
15
|
+
- /var/run/docker.sock:/var/run/docker.sock
|
16
|
+
kafka2:
|
17
|
+
image: wurstmeister/kafka:0.8.2.0
|
18
|
+
ports:
|
19
|
+
- "9993:9092"
|
20
|
+
links:
|
21
|
+
- zookeeper:zk
|
22
|
+
environment:
|
23
|
+
KAFKA_ADVERTISED_HOST_NAME: "192.168.42.45"
|
24
|
+
volumes:
|
25
|
+
- /var/run/docker.sock:/var/run/docker.sock
|
26
|
+
kafka3:
|
27
|
+
image: wurstmeister/kafka:0.8.2.0
|
28
|
+
ports:
|
29
|
+
- "9994:9092"
|
30
|
+
links:
|
31
|
+
- zookeeper:zk
|
32
|
+
environment:
|
33
|
+
KAFKA_ADVERTISED_HOST_NAME: "192.168.42.45"
|
34
|
+
volumes:
|
35
|
+
- /var/run/docker.sock:/var/run/docker.sock
|
data/lib/kafka.rb
CHANGED
@@ -1,9 +1,19 @@
|
|
1
1
|
require "kafka/version"
|
2
|
-
require "kafka/
|
3
|
-
require "kafka/producer"
|
2
|
+
require "kafka/client"
|
4
3
|
|
5
4
|
module Kafka
|
5
|
+
Error = Class.new(StandardError)
|
6
|
+
ConnectionError = Class.new(Error)
|
7
|
+
CorruptMessage = Class.new(Error)
|
8
|
+
UnknownError = Class.new(Error)
|
9
|
+
OffsetOutOfRange = Class.new(Error)
|
10
|
+
UnknownTopicOrPartition = Class.new(Error)
|
11
|
+
InvalidMessageSize = Class.new(Error)
|
12
|
+
LeaderNotAvailable = Class.new(Error)
|
13
|
+
NotLeaderForPartition = Class.new(Error)
|
14
|
+
RequestTimedOut = Class.new(Error)
|
15
|
+
|
6
16
|
def self.new(**options)
|
7
|
-
|
17
|
+
Client.new(**options)
|
8
18
|
end
|
9
19
|
end
|
data/lib/kafka/broker.rb
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
require "logger"
|
2
|
+
require "kafka/connection"
|
3
|
+
require "kafka/protocol"
|
4
|
+
|
5
|
+
module Kafka
|
6
|
+
class Broker
|
7
|
+
def initialize(host:, port:, node_id: nil, client_id:, logger:)
|
8
|
+
@host, @port, @node_id = host, port, node_id
|
9
|
+
|
10
|
+
@connection = Connection.new(
|
11
|
+
host: host,
|
12
|
+
port: port,
|
13
|
+
client_id: client_id,
|
14
|
+
logger: logger
|
15
|
+
)
|
16
|
+
|
17
|
+
@logger = logger
|
18
|
+
end
|
19
|
+
|
20
|
+
def to_s
|
21
|
+
"#{@host}:#{@port} (node_id=#{@node_id.inspect})"
|
22
|
+
end
|
23
|
+
|
24
|
+
def fetch_metadata(**options)
|
25
|
+
api_key = Protocol::TOPIC_METADATA_API_KEY
|
26
|
+
request = Protocol::TopicMetadataRequest.new(**options)
|
27
|
+
response_class = Protocol::MetadataResponse
|
28
|
+
|
29
|
+
response = @connection.request(api_key, request, response_class)
|
30
|
+
|
31
|
+
response.topics.each do |topic|
|
32
|
+
Protocol.handle_error(topic.topic_error_code)
|
33
|
+
|
34
|
+
topic.partitions.each do |partition|
|
35
|
+
Protocol.handle_error(partition.partition_error_code)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
response
|
40
|
+
end
|
41
|
+
|
42
|
+
def produce(**options)
|
43
|
+
api_key = Protocol::PRODUCE_API_KEY
|
44
|
+
request = Protocol::ProduceRequest.new(**options)
|
45
|
+
response_class = request.requires_acks? ? Protocol::ProduceResponse : nil
|
46
|
+
|
47
|
+
@connection.request(api_key, request, response_class)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
require "kafka/broker"
|
2
|
+
|
3
|
+
module Kafka
|
4
|
+
|
5
|
+
# A broker pool represents the set of brokers in a cluster. It needs to be initialized
|
6
|
+
# with a non-empty list of seed brokers. The first seed broker that the pool can connect
|
7
|
+
# to will be asked for the cluster metadata, which allows the pool to map topic
|
8
|
+
# partitions to the current leader for those partitions.
|
9
|
+
class BrokerPool
|
10
|
+
|
11
|
+
# Initializes a broker pool with a set of seed brokers.
|
12
|
+
#
|
13
|
+
# The pool will try to fetch cluster metadata from one of the brokers.
|
14
|
+
#
|
15
|
+
# @param seed_brokers [Array<String>]
|
16
|
+
# @param client_id [String]
|
17
|
+
# @param logger [Logger]
|
18
|
+
def initialize(seed_brokers:, client_id:, logger:)
|
19
|
+
@client_id = client_id
|
20
|
+
@logger = logger
|
21
|
+
@brokers = {}
|
22
|
+
|
23
|
+
initialize_from_seed_brokers(seed_brokers)
|
24
|
+
end
|
25
|
+
|
26
|
+
# Gets the leader of the given topic and partition.
|
27
|
+
#
|
28
|
+
# @param topic [String]
|
29
|
+
# @param partition [Integer]
|
30
|
+
# @return [Broker] the broker that's currently acting as leader of the partition.
|
31
|
+
def get_leader(topic, partition)
|
32
|
+
leader_id = @cluster_info.find_leader_id(topic, partition)
|
33
|
+
|
34
|
+
broker_for_id(leader_id)
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def broker_for_id(broker_id)
|
40
|
+
@brokers[broker_id] ||= connect_to_broker(broker_id)
|
41
|
+
end
|
42
|
+
|
43
|
+
def connect_to_broker(broker_id)
|
44
|
+
broker_info = @cluster_info.find_broker(broker_id)
|
45
|
+
|
46
|
+
Broker.new(
|
47
|
+
host: broker_info.host,
|
48
|
+
port: broker_info.port,
|
49
|
+
node_id: broker_info.node_id,
|
50
|
+
client_id: @client_id,
|
51
|
+
logger: @logger,
|
52
|
+
)
|
53
|
+
end
|
54
|
+
|
55
|
+
def initialize_from_seed_brokers(seed_brokers)
|
56
|
+
seed_brokers.each do |node|
|
57
|
+
@logger.info "Trying to initialize broker pool from node #{node}"
|
58
|
+
|
59
|
+
begin
|
60
|
+
host, port = node.split(":", 2)
|
61
|
+
|
62
|
+
broker = Broker.new(host: host, port: port, client_id: @client_id, logger: @logger)
|
63
|
+
|
64
|
+
@cluster_info = broker.fetch_metadata
|
65
|
+
|
66
|
+
@logger.info "Initialized broker pool with brokers: #{@cluster_info.brokers.inspect}"
|
67
|
+
|
68
|
+
return
|
69
|
+
rescue Error => e
|
70
|
+
@logger.error "Failed to fetch metadata from broker #{broker}: #{e}"
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
raise ConnectionError, "Could not connect to any of the seed brokers: #{seed_brokers.inspect}"
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
data/lib/kafka/client.rb
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
require "kafka/broker_pool"
|
2
|
+
require "kafka/producer"
|
3
|
+
|
4
|
+
module Kafka
|
5
|
+
class Client
|
6
|
+
def initialize(seed_brokers:, client_id:, logger:)
|
7
|
+
@seed_brokers = seed_brokers
|
8
|
+
@client_id = client_id
|
9
|
+
@logger = logger
|
10
|
+
end
|
11
|
+
|
12
|
+
def get_producer(**options)
|
13
|
+
broker_pool = BrokerPool.new(
|
14
|
+
seed_brokers: @seed_brokers,
|
15
|
+
client_id: @client_id,
|
16
|
+
logger: @logger
|
17
|
+
)
|
18
|
+
|
19
|
+
Producer.new(broker_pool: broker_pool, logger: @logger, **options)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
data/lib/kafka/connection.rb
CHANGED
@@ -4,64 +4,109 @@ require "kafka/protocol/encoder"
|
|
4
4
|
require "kafka/protocol/decoder"
|
5
5
|
|
6
6
|
module Kafka
|
7
|
-
ConnectionError = Class.new(StandardError)
|
8
7
|
|
8
|
+
# A connection to a single Kafka broker.
|
9
|
+
#
|
10
|
+
# Usually you'll need a separate connection to each broker in a cluster, since most
|
11
|
+
# requests must be directed specifically to the broker that is currently leader for
|
12
|
+
# the set of topic partitions you want to produce to or consumer from.
|
9
13
|
class Connection
|
14
|
+
API_VERSION = 0
|
15
|
+
|
16
|
+
# Opens a connection to a Kafka broker.
|
17
|
+
#
|
18
|
+
# @param host [String] the hostname of the broker.
|
19
|
+
# @param port [Integer] the port of the broker.
|
20
|
+
# @param client_id [String] the client id is a user-specified string sent in each
|
21
|
+
# request to help trace calls and should logically identify the application
|
22
|
+
# making the request.
|
23
|
+
# @param logger [Logger] the logger used to log trace messages.
|
24
|
+
#
|
25
|
+
# @return [Connection] a new connection.
|
10
26
|
def initialize(host:, port:, client_id:, logger:)
|
11
|
-
@host = host
|
12
|
-
@port = port
|
13
|
-
@client_id = client_id
|
27
|
+
@host, @port, @client_id = host, port, client_id
|
14
28
|
@logger = logger
|
15
|
-
@socket = nil
|
16
|
-
@correlation_id = 0
|
17
|
-
@socket_timeout = 1000
|
18
|
-
end
|
19
29
|
|
20
|
-
def open
|
21
30
|
@logger.info "Opening connection to #{@host}:#{@port} with client id #{@client_id}..."
|
22
31
|
|
23
|
-
@socket = TCPSocket.new(
|
32
|
+
@socket = TCPSocket.new(host, port)
|
33
|
+
|
34
|
+
@encoder = Kafka::Protocol::Encoder.new(@socket)
|
35
|
+
@decoder = Kafka::Protocol::Decoder.new(@socket)
|
36
|
+
|
37
|
+
# Correlation id is initialized to zero and bumped for each request.
|
38
|
+
@correlation_id = 0
|
24
39
|
rescue SocketError => e
|
25
|
-
@logger.error "Failed to connect to #{
|
40
|
+
@logger.error "Failed to connect to #{host}:#{port}: #{e}"
|
26
41
|
|
27
42
|
raise ConnectionError, e
|
28
43
|
end
|
29
44
|
|
45
|
+
def to_s
|
46
|
+
"#{@host}:#{@port}"
|
47
|
+
end
|
48
|
+
|
49
|
+
# Sends a request over the connection.
|
50
|
+
#
|
51
|
+
# @param api_key [Integer] the integer code for the API that is invoked.
|
52
|
+
# @param request [#encode] the request that should be encoded and written.
|
53
|
+
# @param response_class [#decode] an object that can decode the response.
|
54
|
+
#
|
55
|
+
# @return [Object] the response that was decoded by `response_class`.
|
56
|
+
def request(api_key, request, response_class)
|
57
|
+
write_request(api_key, request)
|
58
|
+
|
59
|
+
unless response_class.nil?
|
60
|
+
read_response(response_class)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
private
|
65
|
+
|
66
|
+
# Writes a request over the connection.
|
67
|
+
#
|
68
|
+
# @param api_key [Integer] the integer code for the API that is invoked.
|
69
|
+
# @param request [#encode] the request that should be encoded and written.
|
70
|
+
#
|
71
|
+
# @return [nil]
|
30
72
|
def write_request(api_key, request)
|
31
73
|
@correlation_id += 1
|
74
|
+
@logger.debug "Sending request #{@correlation_id} to #{to_s}"
|
32
75
|
|
33
76
|
message = Kafka::Protocol::RequestMessage.new(
|
34
77
|
api_key: api_key,
|
35
|
-
api_version:
|
78
|
+
api_version: API_VERSION,
|
36
79
|
correlation_id: @correlation_id,
|
37
80
|
client_id: @client_id,
|
38
81
|
request: request,
|
39
82
|
)
|
40
83
|
|
41
|
-
|
42
|
-
|
43
|
-
message.encode(message_encoder)
|
44
|
-
|
45
|
-
@logger.info "Sending request #{@correlation_id} (#{request.class})..."
|
84
|
+
data = Kafka::Protocol::Encoder.encode_with(message)
|
85
|
+
@encoder.write_bytes(data)
|
46
86
|
|
47
|
-
|
48
|
-
connection_encoder.write_bytes(buffer.string)
|
87
|
+
nil
|
49
88
|
end
|
50
89
|
|
51
|
-
|
52
|
-
|
90
|
+
# Reads a response from the connection.
|
91
|
+
#
|
92
|
+
# @param response_class [#decode] an object that can decode the response from
|
93
|
+
# a given Decoder.
|
94
|
+
#
|
95
|
+
# @return [nil]
|
96
|
+
def read_response(response_class)
|
97
|
+
@logger.debug "Waiting for response #{@correlation_id} from #{to_s}"
|
53
98
|
|
54
|
-
|
55
|
-
bytes = connection_decoder.bytes
|
99
|
+
bytes = @decoder.bytes
|
56
100
|
|
57
101
|
buffer = StringIO.new(bytes)
|
58
102
|
response_decoder = Kafka::Protocol::Decoder.new(buffer)
|
59
103
|
|
60
104
|
correlation_id = response_decoder.int32
|
105
|
+
response = response_class.decode(response_decoder)
|
61
106
|
|
62
|
-
@logger.
|
107
|
+
@logger.debug "Received response #{correlation_id} from #{to_s}"
|
63
108
|
|
64
|
-
response
|
109
|
+
response
|
65
110
|
end
|
66
111
|
end
|
67
112
|
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require "kafka/protocol/message"
|
2
|
+
|
3
|
+
module Kafka
|
4
|
+
class MessageSet
|
5
|
+
def initialize(messages)
|
6
|
+
@messages = messages
|
7
|
+
end
|
8
|
+
|
9
|
+
def to_h
|
10
|
+
hsh = {}
|
11
|
+
|
12
|
+
@messages.each do |message|
|
13
|
+
value, key = message.value, message.key
|
14
|
+
topic, partition = message.topic, message.partition
|
15
|
+
|
16
|
+
hsh[topic] ||= {}
|
17
|
+
hsh[topic][partition] ||= []
|
18
|
+
hsh[topic][partition] << Protocol::Message.new(value: value, key: key)
|
19
|
+
end
|
20
|
+
|
21
|
+
hsh
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
data/lib/kafka/producer.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
|
-
require "kafka/
|
1
|
+
require "kafka/message"
|
2
|
+
require "kafka/message_set"
|
2
3
|
|
3
4
|
module Kafka
|
4
5
|
class Producer
|
@@ -6,30 +7,49 @@ module Kafka
|
|
6
7
|
# acknowledgement from the broker before timing out.
|
7
8
|
# @param required_acks [Integer] The number of replicas that must acknowledge
|
8
9
|
# a write.
|
9
|
-
def initialize(
|
10
|
-
@
|
10
|
+
def initialize(broker_pool:, logger:, timeout: 10_000, required_acks: 1)
|
11
|
+
@broker_pool = broker_pool
|
11
12
|
@logger = logger
|
12
13
|
@required_acks = required_acks
|
13
14
|
@timeout = timeout
|
14
|
-
@
|
15
|
+
@buffered_messages = []
|
15
16
|
end
|
16
17
|
|
17
18
|
def write(value, key:, topic:, partition:)
|
18
|
-
|
19
|
-
|
20
|
-
@buffer[topic] ||= {}
|
21
|
-
@buffer[topic][partition] ||= []
|
22
|
-
@buffer[topic][partition] << message
|
19
|
+
@buffered_messages << Message.new(value, key: key, topic: topic, partition: partition)
|
23
20
|
end
|
24
21
|
|
25
22
|
def flush
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
23
|
+
messages_for_broker = {}
|
24
|
+
|
25
|
+
@buffered_messages.each do |message|
|
26
|
+
broker = @broker_pool.get_leader(message.topic, message.partition)
|
27
|
+
|
28
|
+
messages_for_broker[broker] ||= []
|
29
|
+
messages_for_broker[broker] << message
|
30
|
+
end
|
31
|
+
|
32
|
+
messages_for_broker.each do |broker, messages|
|
33
|
+
@logger.info "Sending #{messages.count} messages to broker #{broker}"
|
34
|
+
|
35
|
+
message_set = MessageSet.new(messages)
|
36
|
+
|
37
|
+
response = broker.produce(
|
38
|
+
messages_for_topics: message_set.to_h,
|
39
|
+
required_acks: @required_acks,
|
40
|
+
timeout: @timeout,
|
41
|
+
)
|
42
|
+
|
43
|
+
if response
|
44
|
+
response.topics.each do |topic_info|
|
45
|
+
topic_info.partitions.each do |partition_info|
|
46
|
+
Protocol.handle_error(partition_info.error_code)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
31
51
|
|
32
|
-
@
|
52
|
+
@buffered_messages.clear
|
33
53
|
end
|
34
54
|
end
|
35
55
|
end
|
data/lib/kafka/protocol.rb
CHANGED
@@ -2,6 +2,21 @@ module Kafka
|
|
2
2
|
module Protocol
|
3
3
|
PRODUCE_API_KEY = 0
|
4
4
|
TOPIC_METADATA_API_KEY = 3
|
5
|
+
|
6
|
+
def self.handle_error(error_code)
|
7
|
+
case error_code
|
8
|
+
when -1 then raise UnknownError
|
9
|
+
when 0 then nil # no error, yay!
|
10
|
+
when 1 then raise OffsetOutOfRange
|
11
|
+
when 2 then raise CorruptMessage
|
12
|
+
when 3 then raise UnknownTopicOrPartition
|
13
|
+
when 4 then raise InvalidMessageSize
|
14
|
+
when 5 then raise LeaderNotAvailable
|
15
|
+
when 6 then raise NotLeaderForPartition
|
16
|
+
when 7 then raise RequestTimedOut
|
17
|
+
else raise UnknownError, "Unknown error with code #{error_code}"
|
18
|
+
end
|
19
|
+
end
|
5
20
|
end
|
6
21
|
end
|
7
22
|
|
@@ -1,31 +1,62 @@
|
|
1
1
|
module Kafka
|
2
2
|
module Protocol
|
3
|
+
|
4
|
+
# A decoder wraps an IO object, making it easy to read specific data types
|
5
|
+
# from it. The Kafka protocol is not self-describing, so a client must call
|
6
|
+
# these methods in just the right order for things to work.
|
3
7
|
class Decoder
|
8
|
+
|
9
|
+
# Initializes a new decoder.
|
10
|
+
#
|
11
|
+
# @param io [IO] an object that acts as an IO.
|
4
12
|
def initialize(io)
|
5
13
|
@io = io
|
6
14
|
end
|
7
15
|
|
16
|
+
# Decodes an 8-bit integer from the IO object.
|
17
|
+
#
|
18
|
+
# @return [Integer]
|
8
19
|
def int8
|
9
20
|
read(1).unpack("C").first
|
10
21
|
end
|
11
22
|
|
23
|
+
# Decodes a 16-bit integer from the IO object.
|
24
|
+
#
|
25
|
+
# @return [Integer]
|
12
26
|
def int16
|
13
27
|
read(2).unpack("s>").first
|
14
28
|
end
|
15
29
|
|
30
|
+
# Decodes a 32-bit integer from the IO object.
|
31
|
+
#
|
32
|
+
# @return [Integer]
|
16
33
|
def int32
|
17
34
|
read(4).unpack("l>").first
|
18
35
|
end
|
19
36
|
|
37
|
+
# Decodes a 64-bit integer from the IO object.
|
38
|
+
#
|
39
|
+
# @return [Integer]
|
20
40
|
def int64
|
21
41
|
read(8).unpack("q>").first
|
22
42
|
end
|
23
43
|
|
24
|
-
|
44
|
+
# Decodes an array from the IO object.
|
45
|
+
#
|
46
|
+
# The provided block will be called once for each item in the array. It is
|
47
|
+
# the responsibility of the block to decode the proper type in the block,
|
48
|
+
# since there's no information that allows the type to be inferred
|
49
|
+
# automatically.
|
50
|
+
#
|
51
|
+
# @return [Array]
|
52
|
+
def array(&block)
|
25
53
|
size = int32
|
26
|
-
size.times.map
|
54
|
+
size.times.map(&block)
|
27
55
|
end
|
28
56
|
|
57
|
+
# Decodes a string from the IO object.
|
58
|
+
#
|
59
|
+
# @return [String]
|
29
60
|
def string
|
30
61
|
size = int16
|
31
62
|
|
@@ -36,6 +67,9 @@ module Kafka
|
|
36
67
|
end
|
37
68
|
end
|
38
69
|
|
70
|
+
# Decodes a list of bytes from the IO object.
|
71
|
+
#
|
72
|
+
# @return [String]
|
39
73
|
def bytes
|
40
74
|
size = int32
|
41
75
|
|
@@ -46,6 +80,10 @@ module Kafka
|
|
46
80
|
end
|
47
81
|
end
|
48
82
|
|
83
|
+
# Reads the specified number of bytes from the IO object, returning them
|
84
|
+
# as a String.
|
85
|
+
#
|
86
|
+
# @return [String]
|
49
87
|
def read(number_of_bytes)
|
50
88
|
@io.read(number_of_bytes)
|
51
89
|
end
|
@@ -1,36 +1,77 @@
|
|
1
1
|
module Kafka
|
2
2
|
module Protocol
|
3
|
+
|
4
|
+
# An encoder wraps an IO object, making it easy to write specific data types
|
5
|
+
# to it.
|
3
6
|
class Encoder
|
7
|
+
|
8
|
+
# Initializes a new encoder.
|
9
|
+
#
|
10
|
+
# @param io [IO] an object that acts as an IO.
|
4
11
|
def initialize(io)
|
5
12
|
@io = io
|
6
13
|
@io.set_encoding(Encoding::BINARY)
|
7
14
|
end
|
8
15
|
|
16
|
+
# Writes bytes directly to the IO object.
|
17
|
+
#
|
18
|
+
# @param bytes [String]
|
19
|
+
# @return [nil]
|
9
20
|
def write(bytes)
|
10
21
|
@io.write(bytes)
|
22
|
+
|
23
|
+
nil
|
11
24
|
end
|
12
25
|
|
26
|
+
# Writes an 8-bit integer to the IO object.
|
27
|
+
#
|
28
|
+
# @param int [Integer]
|
29
|
+
# @return [nil]
|
13
30
|
def write_int8(int)
|
14
31
|
write([int].pack("C"))
|
15
32
|
end
|
16
33
|
|
34
|
+
# Writes a 16-bit integer to the IO object.
|
35
|
+
#
|
36
|
+
# @param int [Integer]
|
37
|
+
# @return [nil]
|
17
38
|
def write_int16(int)
|
18
39
|
write([int].pack("s>"))
|
19
40
|
end
|
20
41
|
|
42
|
+
# Writes a 32-bit integer to the IO object.
|
43
|
+
#
|
44
|
+
# @param int [Integer]
|
45
|
+
# @return [nil]
|
21
46
|
def write_int32(int)
|
22
47
|
write([int].pack("l>"))
|
23
48
|
end
|
24
49
|
|
50
|
+
# Writes a 64-bit integer to the IO object.
|
51
|
+
#
|
52
|
+
# @param int [Integer]
|
53
|
+
# @return [nil]
|
25
54
|
def write_int64(int)
|
26
55
|
write([int].pack("q>"))
|
27
56
|
end
|
28
57
|
|
58
|
+
# Writes an array to the IO object.
|
59
|
+
#
|
60
|
+
# Each item in the specified array will be yielded to the provided block;
|
61
|
+
# it's the responsibility of the block to write those items using the
|
62
|
+
# encoder.
|
63
|
+
#
|
64
|
+
# @param array [Array]
|
65
|
+
# @return [nil]
|
29
66
|
def write_array(array, &block)
|
30
67
|
write_int32(array.size)
|
31
68
|
array.each(&block)
|
32
69
|
end
|
33
70
|
|
71
|
+
# Writes a string to the IO object.
|
72
|
+
#
|
73
|
+
# @param string [String]
|
74
|
+
# @return [nil]
|
34
75
|
def write_string(string)
|
35
76
|
if string.nil?
|
36
77
|
write_int16(-1)
|
@@ -40,6 +81,10 @@ module Kafka
|
|
40
81
|
end
|
41
82
|
end
|
42
83
|
|
84
|
+
# Writes a byte string to the IO object.
|
85
|
+
#
|
86
|
+
# @param bytes [String]
|
87
|
+
# @return [nil]
|
43
88
|
def write_bytes(bytes)
|
44
89
|
if bytes.nil?
|
45
90
|
write_int32(-1)
|
@@ -48,6 +93,19 @@ module Kafka
|
|
48
93
|
write(bytes)
|
49
94
|
end
|
50
95
|
end
|
96
|
+
|
97
|
+
# Encodes an object into a new buffer.
|
98
|
+
#
|
99
|
+
# @param object [#encode] the object that will encode itself.
|
100
|
+
# @return [String] the encoded data.
|
101
|
+
def self.encode_with(object)
|
102
|
+
buffer = StringIO.new
|
103
|
+
encoder = new(buffer)
|
104
|
+
|
105
|
+
object.encode(encoder)
|
106
|
+
|
107
|
+
buffer.string
|
108
|
+
end
|
51
109
|
end
|
52
110
|
end
|
53
111
|
end
|
@@ -1,7 +1,37 @@
|
|
1
1
|
module Kafka
|
2
2
|
module Protocol
|
3
|
+
|
4
|
+
# A response to a {TopicMetadataRequest}.
|
5
|
+
#
|
6
|
+
# The response contains information on the brokers, topics, and partitions in
|
7
|
+
# the cluster.
|
8
|
+
#
|
9
|
+
# * For each broker a node id, host, and port is provided.
|
10
|
+
# * For each topic partition the node id of the broker acting as partition leader,
|
11
|
+
# as well as a list of node ids for the set of replicas, are given. The `isr` list is
|
12
|
+
# the subset of replicas that are "in sync", i.e. have fully caught up with the
|
13
|
+
# leader.
|
14
|
+
#
|
15
|
+
# == API Specification
|
16
|
+
#
|
17
|
+
# MetadataResponse => [Broker][TopicMetadata]
|
18
|
+
# Broker => NodeId Host Port (any number of brokers may be returned)
|
19
|
+
# NodeId => int32
|
20
|
+
# Host => string
|
21
|
+
# Port => int32
|
22
|
+
#
|
23
|
+
# TopicMetadata => TopicErrorCode TopicName [PartitionMetadata]
|
24
|
+
# TopicErrorCode => int16
|
25
|
+
#
|
26
|
+
# PartitionMetadata => PartitionErrorCode PartitionId Leader Replicas Isr
|
27
|
+
# PartitionErrorCode => int16
|
28
|
+
# PartitionId => int32
|
29
|
+
# Leader => int32
|
30
|
+
# Replicas => [int32]
|
31
|
+
# Isr => [int32]
|
32
|
+
#
|
3
33
|
class MetadataResponse
|
4
|
-
class
|
34
|
+
class BrokerInfo
|
5
35
|
attr_reader :node_id, :host, :port
|
6
36
|
|
7
37
|
def initialize(node_id:, host:, port:)
|
@@ -9,9 +39,17 @@ module Kafka
|
|
9
39
|
@host = host
|
10
40
|
@port = port
|
11
41
|
end
|
42
|
+
|
43
|
+
def inspect
|
44
|
+
"#{host}:#{port} (node_id=#{node_id})"
|
45
|
+
end
|
12
46
|
end
|
13
47
|
|
14
48
|
class PartitionMetadata
|
49
|
+
attr_reader :partition_id, :leader
|
50
|
+
|
51
|
+
attr_reader :partition_error_code
|
52
|
+
|
15
53
|
def initialize(partition_error_code:, partition_id:, leader:, replicas:, isr:)
|
16
54
|
@partition_error_code = partition_error_code
|
17
55
|
@partition_id = partition_id
|
@@ -22,6 +60,14 @@ module Kafka
|
|
22
60
|
end
|
23
61
|
|
24
62
|
class TopicMetadata
|
63
|
+
# @return [String] the name of the topic
|
64
|
+
attr_reader :topic_name
|
65
|
+
|
66
|
+
# @return [Array<PartitionMetadata>] the partitions in the topic.
|
67
|
+
attr_reader :partitions
|
68
|
+
|
69
|
+
attr_reader :topic_error_code
|
70
|
+
|
25
71
|
def initialize(topic_error_code:, topic_name:, partitions:)
|
26
72
|
@topic_error_code = topic_error_code
|
27
73
|
@topic_name = topic_name
|
@@ -29,22 +75,65 @@ module Kafka
|
|
29
75
|
end
|
30
76
|
end
|
31
77
|
|
32
|
-
|
78
|
+
# @return [Array<BrokerInfo>] the list of brokers in the cluster.
|
79
|
+
attr_reader :brokers
|
80
|
+
|
81
|
+
# @return [Array<TopicMetadata>] the list of topics in the cluster.
|
82
|
+
attr_reader :topics
|
83
|
+
|
84
|
+
def initialize(brokers:, topics:)
|
85
|
+
@brokers = brokers
|
86
|
+
@topics = topics
|
87
|
+
end
|
88
|
+
|
89
|
+
# Finds the node id of the broker that is acting as leader for the given topic
|
90
|
+
# and partition per this metadata.
|
91
|
+
#
|
92
|
+
# @param topic [String] the name of the topic.
|
93
|
+
# @param partition [Integer] the partition number.
|
94
|
+
# @return [Integer] the node id of the leader.
|
95
|
+
def find_leader_id(topic, partition)
|
96
|
+
topic_info = @topics.find {|t| t.topic_name == topic }
|
97
|
+
|
98
|
+
if topic_info.nil?
|
99
|
+
raise "no topic #{topic}"
|
100
|
+
end
|
101
|
+
|
102
|
+
partition_info = topic_info.partitions.find {|p| p.partition_id == partition }
|
103
|
+
|
104
|
+
if partition_info.nil?
|
105
|
+
raise "no partition #{partition} in topic #{topic}"
|
106
|
+
end
|
107
|
+
|
108
|
+
partition_info.leader
|
109
|
+
end
|
110
|
+
|
111
|
+
# Finds the broker info for the given node id.
|
112
|
+
#
|
113
|
+
# @param node_id [Integer] the node id of the broker.
|
114
|
+
# @return [BrokerInfo] information about the broker.
|
115
|
+
def find_broker(node_id)
|
116
|
+
@brokers.find {|broker| broker.node_id == node_id }
|
117
|
+
end
|
33
118
|
|
34
|
-
|
35
|
-
|
119
|
+
# Decodes a MetadataResponse from a {Decoder} containing response data.
|
120
|
+
#
|
121
|
+
# @param decoder [Decoder]
|
122
|
+
# @return [MetadataResponse] the metadata response.
|
123
|
+
def self.decode(decoder)
|
124
|
+
brokers = decoder.array do
|
36
125
|
node_id = decoder.int32
|
37
126
|
host = decoder.string
|
38
127
|
port = decoder.int32
|
39
128
|
|
40
|
-
|
129
|
+
BrokerInfo.new(
|
41
130
|
node_id: node_id,
|
42
131
|
host: host,
|
43
132
|
port: port
|
44
133
|
)
|
45
134
|
end
|
46
135
|
|
47
|
-
|
136
|
+
topics = decoder.array do
|
48
137
|
topic_error_code = decoder.int16
|
49
138
|
topic_name = decoder.string
|
50
139
|
|
@@ -64,6 +153,8 @@ module Kafka
|
|
64
153
|
partitions: partitions,
|
65
154
|
)
|
66
155
|
end
|
156
|
+
|
157
|
+
new(brokers: brokers, topics: topics)
|
67
158
|
end
|
68
159
|
end
|
69
160
|
end
|
@@ -74,7 +74,7 @@ module Kafka
|
|
74
74
|
# When encoding a message into a message set, the bytesize of the message must
|
75
75
|
# precede the actual bytes. Therefore we need to encode the message into a
|
76
76
|
# separate buffer first.
|
77
|
-
encoded_message =
|
77
|
+
encoded_message = Encoder.encode_with(message)
|
78
78
|
|
79
79
|
encoder.write_int64(offset)
|
80
80
|
|
@@ -84,15 +84,6 @@ module Kafka
|
|
84
84
|
|
85
85
|
buffer.string
|
86
86
|
end
|
87
|
-
|
88
|
-
def encode_message(message)
|
89
|
-
buffer = StringIO.new
|
90
|
-
encoder = Encoder.new(buffer)
|
91
|
-
|
92
|
-
message.encode(encoder)
|
93
|
-
|
94
|
-
buffer.string
|
95
|
-
end
|
96
87
|
end
|
97
88
|
end
|
98
89
|
end
|
@@ -4,7 +4,7 @@ module Kafka
|
|
4
4
|
class TopicInfo
|
5
5
|
attr_reader :topic, :partitions
|
6
6
|
|
7
|
-
def initialize(topic
|
7
|
+
def initialize(topic:, partitions:)
|
8
8
|
@topic = topic
|
9
9
|
@partitions = partitions
|
10
10
|
end
|
@@ -13,7 +13,7 @@ module Kafka
|
|
13
13
|
class PartitionInfo
|
14
14
|
attr_reader :partition, :error_code, :offset
|
15
15
|
|
16
|
-
def initialize(partition
|
16
|
+
def initialize(partition:, error_code:, offset:)
|
17
17
|
@partition = partition
|
18
18
|
@error_code = error_code
|
19
19
|
@offset = offset
|
@@ -22,20 +22,26 @@ module Kafka
|
|
22
22
|
|
23
23
|
attr_reader :topics
|
24
24
|
|
25
|
-
def
|
26
|
-
@topics =
|
25
|
+
def initialize(topics: [])
|
26
|
+
@topics = topics
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.decode(decoder)
|
30
|
+
topics = decoder.array do
|
27
31
|
topic = decoder.string
|
28
32
|
|
29
33
|
partitions = decoder.array do
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
34
|
+
PartitionInfo.new(
|
35
|
+
partition: decoder.int32,
|
36
|
+
error_code: decoder.int16,
|
37
|
+
offset: decoder.int64,
|
38
|
+
)
|
35
39
|
end
|
36
40
|
|
37
|
-
TopicInfo.new(topic, partitions)
|
41
|
+
TopicInfo.new(topic: topic, partitions: partitions)
|
38
42
|
end
|
43
|
+
|
44
|
+
new(topics: topics)
|
39
45
|
end
|
40
46
|
end
|
41
47
|
end
|
data/lib/kafka/version.rb
CHANGED
data/lib/ruby-kafka.rb
ADDED
data/test-setup.sh
ADDED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ruby-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.0.pre.
|
4
|
+
version: 0.1.0.pre.alpha2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Schierbeck
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-01-
|
11
|
+
date: 2016-01-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -69,10 +69,15 @@ files:
|
|
69
69
|
- Rakefile
|
70
70
|
- bin/console
|
71
71
|
- bin/setup
|
72
|
+
- docker-compose.yml
|
72
73
|
- kafka.gemspec
|
73
74
|
- lib/kafka.rb
|
74
|
-
- lib/kafka/
|
75
|
+
- lib/kafka/broker.rb
|
76
|
+
- lib/kafka/broker_pool.rb
|
77
|
+
- lib/kafka/client.rb
|
75
78
|
- lib/kafka/connection.rb
|
79
|
+
- lib/kafka/message.rb
|
80
|
+
- lib/kafka/message_set.rb
|
76
81
|
- lib/kafka/producer.rb
|
77
82
|
- lib/kafka/protocol.rb
|
78
83
|
- lib/kafka/protocol/decoder.rb
|
@@ -84,6 +89,8 @@ files:
|
|
84
89
|
- lib/kafka/protocol/request_message.rb
|
85
90
|
- lib/kafka/protocol/topic_metadata_request.rb
|
86
91
|
- lib/kafka/version.rb
|
92
|
+
- lib/ruby-kafka.rb
|
93
|
+
- test-setup.sh
|
87
94
|
homepage: https://github.com/zendesk/ruby-kafka
|
88
95
|
licenses:
|
89
96
|
- Apache License Version 2.0
|
data/lib/kafka/cluster.rb
DELETED
@@ -1,53 +0,0 @@
|
|
1
|
-
require "logger"
|
2
|
-
require "kafka/connection"
|
3
|
-
require "kafka/protocol"
|
4
|
-
|
5
|
-
module Kafka
|
6
|
-
class Cluster
|
7
|
-
def self.connect(brokers:, client_id:, logger:)
|
8
|
-
host, port = brokers.first.split(":", 2)
|
9
|
-
|
10
|
-
connection = Connection.new(
|
11
|
-
host: host,
|
12
|
-
port: port.to_i,
|
13
|
-
client_id: client_id,
|
14
|
-
logger: logger
|
15
|
-
)
|
16
|
-
|
17
|
-
connection.open
|
18
|
-
|
19
|
-
new(connection: connection, logger: logger)
|
20
|
-
end
|
21
|
-
|
22
|
-
def initialize(connection:, logger: nil)
|
23
|
-
@connection = connection
|
24
|
-
@logger = logger
|
25
|
-
end
|
26
|
-
|
27
|
-
def fetch_metadata(**options)
|
28
|
-
api_key = Protocol::TOPIC_METADATA_API_KEY
|
29
|
-
request = Protocol::TopicMetadataRequest.new(**options)
|
30
|
-
response = Protocol::MetadataResponse.new
|
31
|
-
|
32
|
-
@connection.write_request(api_key, request)
|
33
|
-
@connection.read_response(response)
|
34
|
-
|
35
|
-
response
|
36
|
-
end
|
37
|
-
|
38
|
-
def produce(**options)
|
39
|
-
api_key = Protocol::PRODUCE_API_KEY
|
40
|
-
request = Protocol::ProduceRequest.new(**options)
|
41
|
-
|
42
|
-
@connection.write_request(api_key, request)
|
43
|
-
|
44
|
-
if request.requires_acks?
|
45
|
-
response = Protocol::ProduceResponse.new
|
46
|
-
@connection.read_response(response)
|
47
|
-
response
|
48
|
-
else
|
49
|
-
nil
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|