codeclimate-poseidon 0.0.8

data/lib/poseidon/protocol/request_buffer.rb

@@ -0,0 +1,77 @@
+module Poseidon
+  module Protocol
+    # RequestBuffer allows you to build a Binary string for API requests
+    #
+    # API parallels the primitive types described on the wiki, with some
+    # sugar for prepending message sizes and checksums.
+    # (https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ProtocolPrimitiveTypes)
+    class RequestBuffer
+      def initialize
+        @s = ''.encode(Encoding::BINARY)
+      end
+
+      def append(string)
+        string = string.dup.force_encoding(Encoding::BINARY)
+        @s << string
+        nil
+      end
+
+      def int8(int8)
+        append([int8].pack("C"))
+      end
+
+      def int16(int16)
+        append([int16].pack("s>"))
+      end
+
+      def int32(int32)
+        append([int32].pack("l>"))
+      end
+
+      def int64(int64)
+        append([int64].pack("q>"))
+      end
+
+      # Add a string
+      #
+      # @param [String] string
+      def string(string)
+        if string.nil?
+          int16(-1)
+        else
+          int16(string.bytesize)
+          append(string)
+        end
+      end
+
+      def bytes(string)
+        if string.nil?
+          int32(-1)
+        else
+          int32(string.bytesize)
+          append(string)
+        end
+      end
+
+      def prepend_crc32
+        checksum_pos = @s.bytesize
+        @s += " "
+        yield
+        @s[checksum_pos] = [Zlib::crc32(@s[(checksum_pos+1)..-1])].pack("N")
+        nil
+      end
+
+      def prepend_size
+        size_pos = @s.bytesize
+        @s += " "
+        yield
+        @s[size_pos] = [(@s.bytesize-1) - size_pos].pack("N")
+        nil
+      end
+
+      def to_s
+        @s
+      end
+    end
+  end
+end

data/lib/poseidon/protocol/response_buffer.rb

@@ -0,0 +1,72 @@
+module Poseidon
+  module Protocol
+    class ResponseBuffer
+      def initialize(response)
+        @s = response
+        @pos = 0
+      end
+
+      def int8
+        byte = @s.byteslice(@pos, 1).unpack("C").first
+        @pos += 1
+        byte
+      end
+
+      def int16
+        short = @s.byteslice(@pos, 2).unpack("s>").first
+        @pos += 2
+        short
+      end
+
+      def int32
+        int = @s.byteslice(@pos, 4).unpack("l>").first
+        @pos += 4
+        int
+      end
+
+      def int64
+        long = @s.byteslice(@pos, 8).unpack("q>").first
+        @pos += 8
+        long
+      end
+
+      def string
+        len = int16
+        string = @s.byteslice(@pos, len)
+        @pos += len
+        string
+      end
+
+      def read(bytes)
+        data = @s.byteslice(@pos, bytes)
+        @pos += bytes
+        data
+      end
+
+      def peek(bytes)
+        @s.byteslice(@pos, bytes)
+      end
+
+      def bytes
+        n = int32
+        if n == -1
+          return nil
+        else
+          read(n)
+        end
+      end
+
+      def bytes_remaining
+        @s.bytesize - @pos
+      end
+
+      def eof?
+        @pos == @s.bytesize
+      end
+
+      def to_s
+        @s
+      end
+    end
+  end
+end

data/lib/poseidon/sync_producer.rb

@@ -0,0 +1,161 @@
+module Poseidon
+  # Used by +Producer+ for sending messages to the kafka cluster.
+  #
+  # You should not use this interface directly
+  #
+  # Fetches metadata at appropriate times.
+  # Builds MessagesToSend
+  # Handle MessageBatchToSend lifecyle
+  #
+  # Who is responsible for fetching metadata from broker seed list?
+  #   Do we want to be fetching from real live brokers eventually?
+  #
+  # @api private
+  class SyncProducer
+    OPTION_DEFAULTS = {
+      :compression_codec => nil,
+      :compressed_topics => nil,
+      :metadata_refresh_interval_ms => 600_000,
+      :partitioner => nil,
+      :max_send_retries => 3,
+      :retry_backoff_ms => 100,
+      :required_acks => 0,
+      :ack_timeout_ms => 1500,
+      :socket_timeout_ms => 10_000
+    }
+
+    attr_reader :client_id, :retry_backoff_ms, :max_send_retries,
+      :metadata_refresh_interval_ms, :required_acks, :ack_timeout_ms, :socket_timeout_ms
+    def initialize(client_id, seed_brokers, options = {})
+      @client_id = client_id
+
+      handle_options(options.dup)
+
+      @cluster_metadata   = ClusterMetadata.new
+      @message_conductor  = MessageConductor.new(@cluster_metadata, @partitioner)
+      @broker_pool        = BrokerPool.new(client_id, seed_brokers, socket_timeout_ms)
+    end
+
+    def send_messages(messages)
+      return if messages.empty?
+
+      messages_to_send = MessagesToSend.new(messages, @cluster_metadata)
+
+      if refresh_interval_elapsed?
+        refresh_metadata(messages_to_send.topic_set)
+      end
+
+      ensure_metadata_available_for_topics(messages_to_send)
+
+      (@max_send_retries+1).times do
+        messages_to_send.messages_for_brokers(@message_conductor).each do |messages_for_broker|
+          if sent = send_to_broker(messages_for_broker)
+            messages_to_send.successfully_sent(sent)
+          end
+        end
+
+        if !messages_to_send.pending_messages? || @max_send_retries == 0
+          break
+        else
+          Kernel.sleep retry_backoff_ms / 1000.0
+          reset_metadata
+          ensure_metadata_available_for_topics(messages_to_send)
+        end
+      end
+
+      if messages_to_send.pending_messages?
+        raise "Failed to send all messages: #{messages_to_send.messages} remaining"
+      else
+        true
+      end
+    end
+
+    def close
+      @broker_pool.close
+    end
+
+    alias_method :shutdown, :close
+
+    private
+
+    def ensure_metadata_available_for_topics(messages_to_send)
+      return if !messages_to_send.needs_metadata?
+
+      Poseidon.logger.debug { "Fetching metadata for #{messages_to_send.topic_set.inspect}. (Attempt 1)" }
+      refresh_metadata(messages_to_send.topic_set)
+      return if !messages_to_send.needs_metadata?
+
+      2.times do |n|
+        sleep 5
+
+        Poseidon.logger.debug { "Fetching metadata for #{messages_to_send.topic_set.inspect}. (Attempt #{n+2})" }
+        refresh_metadata(messages_to_send.topic_set)
+        return if !messages_to_send.needs_metadata?
+      end
+      raise Errors::UnableToFetchMetadata
+    end
+
+    def handle_options(options)
+      @ack_timeout_ms    = handle_option(options, :ack_timeout_ms)
+      @socket_timeout_ms = handle_option(options, :socket_timeout_ms)
+      @retry_backoff_ms  = handle_option(options, :retry_backoff_ms)
+
+      @metadata_refresh_interval_ms = 
+        handle_option(options, :metadata_refresh_interval_ms)
+
+      @required_acks    = handle_option(options, :required_acks)
+      @max_send_retries = handle_option(options, :max_send_retries)
+
+      @compression_config = ProducerCompressionConfig.new(
+        handle_option(options, :compression_codec), 
+        handle_option(options, :compressed_topics))
+
+      @partitioner = handle_option(options, :partitioner)
+
+      raise ArgumentError, "Unknown options: #{options.keys.inspect}" if options.keys.any?
+    end
+
+    def handle_option(options, sym)
+      options.delete(sym) || OPTION_DEFAULTS[sym]
+    end
+
+    def refresh_interval_elapsed?
+      @cluster_metadata.last_refreshed_at.nil? ||
+        (Time.now - @cluster_metadata.last_refreshed_at) * 1000 > metadata_refresh_interval_ms
+    end
+
+    def refresh_metadata(topics)
+      topics_to_refresh = topics.dup
+
+      @cluster_metadata.topics.each do |topic|
+        topics_to_refresh.add(topic)
+      end
+
+      @cluster_metadata.update(@broker_pool.fetch_metadata(topics_to_refresh))
+      @broker_pool.update_known_brokers(@cluster_metadata.brokers)
+    end
+
+    def reset_metadata
+      Poseidon.logger.debug { "Resetting metdata" }
+      @cluster_metadata.reset
+      @broker_pool.close
+    end
+
+    def send_to_broker(messages_for_broker)
+      return false if messages_for_broker.broker_id == -1
+      to_send = messages_for_broker.build_protocol_objects(@compression_config)
+
+      Poseidon.logger.debug { "Sending messages to broker #{messages_for_broker.broker_id}" }
+      response = @broker_pool.execute_api_call(messages_for_broker.broker_id, :produce,
+                                              required_acks, ack_timeout_ms,
+                                              to_send)
+      if required_acks == 0
+        messages_for_broker.messages
+      else
+        messages_for_broker.successfully_sent(response)
+      end
+    rescue Connection::ConnectionFailedError
+      false
+    end
+  end
+end

data/lib/poseidon/topic_metadata.rb

@@ -0,0 +1,89 @@
+module Poseidon
+  # @api private
+  class TopicMetadata
+    # Build a new TopicMetadata object from its binary representation
+    #
+    # @param [ResponseBuffer] buffer
+    # @return [TopicMetadata]
+    #
+    def self.read(buffer)
+      tm = TopicMetadata.new
+      tm.struct = Protocol::TopicMetadataStruct.read(buffer)
+      tm
+    end
+
+    attr_accessor :struct
+    def initialize(struct=nil)
+      self.struct = struct
+    end
+
+    # Write a binary representation of the TopicMetadata to buffer
+    # 
+    # @param [RequestBuffer] buffer
+    # @return [nil]
+    def write(buffer)
+      struct.write(buffer)
+      nil
+    end
+
+    def name
+      struct.name
+    end
+
+    def ==(o)
+      eql?(o)
+    end
+
+    def exists?
+      struct.error == Errors::NO_ERROR_CODE
+    end
+
+    def eql?(o)
+      struct.eql?(o.struct)
+    end
+
+    def objects_with_errors
+      struct.objects_with_errors
+    end
+
+    def leader_available?
+      struct.error_class != Errors::LeaderNotAvailable
+    end
+
+    def partition_count
+      @partition_count ||= struct.partitions.count
+    end
+
+    def available_partitions
+      @available_partitions ||= struct.partitions.select do |partition|
+        (partition.error == Errors::NO_ERROR_CODE || partition.error_class == Errors::ReplicaNotAvailable) && partition.leader != -1
+      end
+    end
+
+    def available_partition_count
+      available_partitions.count
+    end
+
+    def partition_leader(partition_id)
+      partition = partitions_by_id[partition_id]
+      if partition
+        partition.leader
+      else
+        nil
+      end
+    end
+
+    def to_s
+      struct.partitions.map { |p| p.inspect }.join("\n")
+    end
+
+    private
+    def partitions_by_id
+      @partitions_by_id ||= Hash[partitions.map { |p| [p.id, p] }]
+    end
+
+    def partitions
+      struct.partitions
+    end
+  end
+end

data/lib/poseidon/version.rb

@@ -0,0 +1,4 @@
+module Poseidon
+  # Unstable! API May Change!
+  VERSION = "0.0.8"
+end

data/poseidon.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'poseidon/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "codeclimate-poseidon"
+  gem.version       = Poseidon::VERSION
+  gem.authors       = ["Bob Potter"]
+  gem.email         = ["bobby.potter@gmail.com"]
+  gem.description   = %q{A Kafka (http://kafka.apache.org/) producer and consumer}
+  gem.summary       = %q{Poseidon is a producer and consumer implementation for Kafka >= 0.8}
+  gem.homepage      = "https://github.com/bpot/poseidon"
+  gem.licenses      = ["MIT"]
+  gem.required_ruby_version = '>= 1.9.3'
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_development_dependency(%q<rspec>, '>= 3')
+  gem.add_development_dependency(%q<yard>)
+  gem.add_development_dependency(%q<simplecov>)
+  gem.add_development_dependency(%q<snappy>)
+  gem.add_development_dependency(%q<timecop>)
+end

data/spec/integration/multiple_brokers/consumer_spec.rb

@@ -0,0 +1,45 @@
+require 'integration/multiple_brokers/spec_helper'
+
+RSpec.describe "consuming with multiple brokers", :type => :request do
+  include_context "a multiple broker cluster"
+
+  before(:each) do
+    # autocreate the topic by asking for information about it
+    c = Connection.new("localhost", 9092, "metadata_fetcher", 10_000)
+    md = c.topic_metadata(["test"])
+    sleep 1
+  end
+
+  it "finds the lead broker for each partition" do
+    brokers = Set.new
+    0.upto(2) do |partition|
+      pc = PartitionConsumer.consumer_for_partition("test_client",
+                                                    ["localhost:9092"],
+                                                    "test", partition,
+                                                    :earliest_offset)
+
+      brokers.add("#{pc.host}:#{pc.port}")
+    end
+    expect(brokers.size).to eq(3)
+  end
+
+  it "consumes from all partitions" do
+    @p = Producer.new(["localhost:9092","localhost:9093","localhost:9094"], "test",
+                     :required_acks => 1)
+
+    msgs = 24.times.map { |n| "hello_#{n}" }
+    msgs.each do |msg|
+      @p.send_messages([MessageToSend.new("test", msg)])
+    end
+
+    fetched_messages = []
+    0.upto(2) do |partition|
+      pc = PartitionConsumer.consumer_for_partition("test_client",
+                                                    ["localhost:9092"],
+                                                    "test", partition,
+                                                    :earliest_offset)
+      fetched_messages.push(*pc.fetch)
+    end
+    expect(fetched_messages.map(&:value).sort).to eq(msgs.sort)
+  end
+end