ruby-kafka 0.1.3 → 0.1.4

data/lib/kafka/protocol/message.rb

@@ -36,6 +36,21 @@ module Kafka
         @key == other.key && @value == other.value && @attributes == other.attributes
       end
 
+      def self.decode(decoder)
+        crc = decoder.int32
+        magic_byte = decoder.int8
+
+        unless magic_byte == MAGIC_BYTE
+          raise Kafka::Error, "Invalid magic byte: #{magic_byte}"
+        end
+
+        attributes = decoder.int8
+        key = decoder.bytes
+        value = decoder.bytes
+
+        new(key: key, value: value, attributes: attributes)
+      end
+
       private
 
       def encode_without_crc

data/lib/kafka/protocol/message_set.rb

@@ -0,0 +1,25 @@
+module Kafka
+  module Protocol
+    class MessageSet
+      attr_reader :messages
+
+      def initialize(messages:)
+        @messages = messages
+      end
+
+      def self.decode(decoder)
+        fetched_messages = []
+
+        until decoder.eof?
+          offset = decoder.int64
+          message_decoder = Decoder.from_string(decoder.bytes)
+          message = Message.decode(message_decoder)
+
+          fetched_messages << [offset, message]
+        end
+
+        new(messages: fetched_messages)
+      end
+    end
+  end
+end

data/lib/kafka/protocol/metadata_response.rb

@@ -40,7 +40,7 @@ module Kafka
           @port = port
         end
 
-        def inspect
+        def to_s
           "#{host}:#{port} (node_id=#{node_id})"
         end
       end
@@ -131,6 +131,8 @@ module Kafka
           raise UnknownTopicOrPartition, "unknown topic #{topic_name}"
         end
 
+        Protocol.handle_error(topic.topic_error_code)
+
         topic.partitions
       end
 

data/lib/kafka/protocol/produce_request.rb

@@ -27,6 +27,9 @@ module Kafka
     class ProduceRequest
       attr_reader :required_acks, :timeout, :messages_for_topics
 
+      # @param required_acks [Integer]
+      # @param timeout [Integer]
+      # @param messages_for_topics [Hash]
       def initialize(required_acks:, timeout:, messages_for_topics:)
         @required_acks = required_acks
         @timeout = timeout

data/lib/kafka/protocol/topic_metadata_request.rb

@@ -1,6 +1,10 @@
 module Kafka
   module Protocol
     class TopicMetadataRequest
+
+      # A request for cluster metadata.
+      #
+      # @param topics [Array<String>]
       def initialize(topics: [])
         @topics = topics
       end

data/lib/kafka/socket_with_timeout.rb

@@ -28,15 +28,15 @@ module Kafka
       @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
 
       begin
-        # Initiate the socket connection in the background. If it doesn't fail 
-        # immediately it will raise an IO::WaitWritable (Errno::EINPROGRESS) 
+        # Initiate the socket connection in the background. If it doesn't fail
+        # immediately it will raise an IO::WaitWritable (Errno::EINPROGRESS)
         # indicating the connection is in progress.
         @socket.connect_nonblock(sockaddr)
       rescue IO::WaitWritable
         # IO.select will block until the socket is writable or the timeout
         # is exceeded, whichever comes first.
         unless IO.select(nil, [@socket], nil, connect_timeout)
-          # IO.select returns nil when the socket is not ready before timeout 
+          # IO.select returns nil when the socket is not ready before timeout
           # seconds have elapsed
           @socket.close
           raise Errno::ETIMEDOUT

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

data/ruby-kafka.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Daniel Schierbeck"]
   spec.email         = ["daniel.schierbeck@gmail.com"]
 
-  spec.summary       = %q{A client library for the Kafka distributed commit log.}
+  spec.summary       = "A client library for the Kafka distributed commit log."
 
   spec.description   = <<-DESC.gsub(/^    /, "").strip
     A client library for the Kafka distributed commit log.
@@ -20,6 +20,8 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://github.com/zendesk/ruby-kafka"
   spec.license       = "Apache License Version 2.0"
 
+  spec.required_ruby_version = '>= 2.1.0'
+
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-02-08 00:00:00.000000000 Z
+date: 2016-02-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -148,27 +148,37 @@ files:
 - bin/console
 - bin/setup
 - circle.yml
+- examples/simple-consumer.rb
 - examples/simple-producer.rb
 - lib/kafka.rb
 - lib/kafka/broker.rb
 - lib/kafka/broker_pool.rb
 - lib/kafka/client.rb
+- lib/kafka/cluster.rb
 - lib/kafka/connection.rb
+- lib/kafka/fetch_operation.rb
+- lib/kafka/fetched_message.rb
 - lib/kafka/instrumentation.rb
 - lib/kafka/message_buffer.rb
 - lib/kafka/partitioner.rb
+- lib/kafka/pending_message.rb
+- lib/kafka/produce_operation.rb
 - lib/kafka/producer.rb
 - lib/kafka/protocol.rb
 - lib/kafka/protocol/decoder.rb
 - lib/kafka/protocol/encoder.rb
+- lib/kafka/protocol/fetch_request.rb
+- lib/kafka/protocol/fetch_response.rb
+- lib/kafka/protocol/list_offset_request.rb
+- lib/kafka/protocol/list_offset_response.rb
 - lib/kafka/protocol/message.rb
+- lib/kafka/protocol/message_set.rb
 - lib/kafka/protocol/metadata_response.rb
 - lib/kafka/protocol/produce_request.rb
 - lib/kafka/protocol/produce_response.rb
 - lib/kafka/protocol/request_message.rb
 - lib/kafka/protocol/topic_metadata_request.rb
 - lib/kafka/socket_with_timeout.rb
-- lib/kafka/transmission.rb
 - lib/kafka/version.rb
 - lib/ruby-kafka.rb
 - ruby-kafka.gemspec
@@ -184,7 +194,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/lib/kafka/transmission.rb

@@ -1,76 +0,0 @@
-module Kafka
-  class Transmission
-    def initialize(broker_pool:, buffer:, required_acks:, ack_timeout:, logger:)
-      @broker_pool = broker_pool
-      @buffer = buffer
-      @required_acks = required_acks
-      @ack_timeout = ack_timeout
-      @logger = logger
-    end
-
-    def send_messages
-      messages_for_broker = {}
-
-      @buffer.each do |topic, partition, messages|
-        broker = @broker_pool.get_leader(topic, partition)
-
-        @logger.debug "Current leader for #{topic}/#{partition} is node #{broker}"
-
-        messages_for_broker[broker] ||= MessageBuffer.new
-        messages_for_broker[broker].concat(messages, topic: topic, partition: partition)
-      end
-
-      messages_for_broker.each do |broker, message_set|
-        begin
-          response = broker.produce(
-            messages_for_topics: message_set.to_h,
-            required_acks: @required_acks,
-            timeout: @ack_timeout * 1000, # Kafka expects the timeout in milliseconds.
-          )
-
-          handle_response(response) if response
-        rescue ConnectionError => e
-          @logger.error "Could not connect to broker #{broker}: #{e}"
-
-          # Mark the broker pool as stale in order to force a cluster metadata refresh.
-          @broker_pool.mark_as_stale!
-        end
-      end
-    end
-
-    private
-
-    def handle_response(response)
-      response.each_partition do |topic_info, partition_info|
-        topic = topic_info.topic
-        partition = partition_info.partition
-
-        begin
-          Protocol.handle_error(partition_info.error_code)
-        rescue Kafka::CorruptMessage
-          @logger.error "Corrupt message when writing to #{topic}/#{partition}"
-        rescue Kafka::UnknownTopicOrPartition
-          @logger.error "Unknown topic or partition #{topic}/#{partition}"
-        rescue Kafka::LeaderNotAvailable
-          @logger.error "Leader currently not available for #{topic}/#{partition}"
-          @broker_pool.mark_as_stale!
-        rescue Kafka::NotLeaderForPartition
-          @logger.error "Broker not currently leader for #{topic}/#{partition}"
-          @broker_pool.mark_as_stale!
-        rescue Kafka::RequestTimedOut
-          @logger.error "Timed out while writing to #{topic}/#{partition}"
-        rescue Kafka::NotEnoughReplicas
-          @logger.error "Not enough in-sync replicas for #{topic}/#{partition}"
-        rescue Kafka::NotEnoughReplicasAfterAppend
-          @logger.error "Messages written, but to fewer in-sync replicas than required for #{topic}/#{partition}"
-        else
-          offset = partition_info.offset
-          @logger.info "Successfully sent messages for #{topic}/#{partition}; new offset is #{offset}"
-
-          # The messages were successfully written; clear them from the buffer.
-          @buffer.clear_messages(topic: topic, partition: partition)
-        end
-      end
-    end
-  end
-end