poseidon 0.0.1

data/lib/poseidon/protocol.rb

@@ -0,0 +1,122 @@
+module Poseidon
+  # @api private
+  module Protocol
+    require "poseidon/protocol/protocol_struct"
+    require "poseidon/protocol/request_buffer"
+    require "poseidon/protocol/response_buffer"
+
+    API_KEYS = {
+      :produce => 0,
+      :fetch => 1,
+      :offset => 2,
+      :metadata => 3
+    }
+
+    # Request/Response Common Structures
+    RequestCommon = ProtocolStruct.new(:api_key => :int16,
+                                       :api_version => :int16,
+                                       :correlation_id => :int32,
+                                       :client_id => :string)
+    ResponseCommon = ProtocolStruct.new(:correlation_id => :int32)
+
+    # MessageSet Common Structure
+    MessageStruct = ProtocolStruct.new(:magic_type => :int8,
+                                       :attributes => :int8,
+                                       :key => :bytes,
+                                       :value => :bytes).prepend_size.prepend_crc32.truncatable
+    MessageWithOffsetStruct = ProtocolStruct.new(:offset => :int64,
+                                                 :message => MessageStruct)
+     
+    # When part of produce requests of fetch responses a MessageSet
+    # has a prepended size.  When a MessageSet is compressed and
+    # nested in a Message size is not prepended.
+    MessageSetStruct = ProtocolStruct.new(:messages => [Message]).
+                                          size_bound_array(:messages)
+    MessageSetStructWithSize = MessageSetStruct.dup.prepend_size
+
+    # Produce Request
+    MessagesForPartition = ProtocolStruct.new(:partition    => :int32,
+                                              :message_set  => MessageSet)
+    MessagesForTopic = ProtocolStruct.new(:topic => :string,
+                                          :messages_for_partitions =>
+                                            [MessagesForPartition])
+    ProduceRequest = ProtocolStruct.new(:common => RequestCommon,
+                                        :required_acks => :int16,
+                                        :timeout => :int32,
+                                        :messages_for_topics => [MessagesForTopic])
+
+    # Produce Response
+    ProducePartitionResponse = ProtocolStruct.new(:partition => :int32,
+                                                  :error => :int16,
+                                                  :offset => :int64)
+    ProduceTopicResponse = ProtocolStruct.new(:topic => :string,
+                                              :partitions => [ProducePartitionResponse])
+    ProduceResponse = ProtocolStruct.new(:common => ResponseCommon,
+                                         :topic_response => [ProduceTopicResponse])
+
+    # Fetch Request
+    PartitionFetch = ProtocolStruct.new(:partition => :int32,
+                                        :fetch_offset => :int64,
+                                        :max_bytes => :int32)
+    TopicFetch = ProtocolStruct.new(:topic => :string,
+                                    :partition_fetches => [PartitionFetch])
+    FetchRequest = ProtocolStruct.new(:common => RequestCommon,
+                                      :replica_id => :int32,
+                                      :max_wait_time => :int32,
+                                      :min_bytes => :int32,
+                                      :topic_fetches => [TopicFetch])
+
+    # Fetch Response
+    PartitionFetchResponse = ProtocolStruct.new(:partition => :int32,
+                                                :error => :int16,
+                                                :highwater_mark_offset => :int64,
+                                                :message_set => MessageSet)
+    TopicFetchResponse = ProtocolStruct.new(:topic => :string,
+                                            :partition_fetch_responses => [PartitionFetchResponse])
+    FetchResponse = ProtocolStruct.new(
+      :common => ResponseCommon,
+      :topic_fetch_responses => [TopicFetchResponse])
+
+    # Offset Request
+    PartitionOffsetRequest = ProtocolStruct.new(:partition => :int32,
+                                                :time => :int64,
+                                                :max_number_of_offsets => :int32)
+    TopicOffsetRequest = ProtocolStruct.new(
+      :topic => :string,
+      :partition_offset_requests => [PartitionOffsetRequest])
+    OffsetRequest = ProtocolStruct.new(:common => RequestCommon,
+                                       :replica_id => :int32,
+                                       :topic_offset_requests => [TopicOffsetRequest])
+
+    # Offset Response
+    Offset = ProtocolStruct.new(:offset => :int64)
+    PartitionOffset = ProtocolStruct.new(:partition => :int32,
+                                         :error => :int16,
+                                         :offsets => [Offset])
+    TopicOffsetResponse = ProtocolStruct.new(:topic => :string,
+                                             :partition_offsets => [PartitionOffset])
+    OffsetResponse = ProtocolStruct.new(
+      :common => ResponseCommon,
+      :topic_offset_responses => [TopicOffsetResponse])
+
+    # Metadata Request
+    MetadataRequest = ProtocolStruct.new( :common => RequestCommon,
+                                          :topic_names => [:string])
+
+    # Metadata Response
+    Broker = ProtocolStruct.new(:id => :int32,
+                                :host => :string,
+                                :port => :int32)
+    PartitionMetadata = ProtocolStruct.new(:error     => :int16,
+                                           :id        => :int32,
+                                           :leader    => :int32,
+                                           :replicas  => [:int32],
+                                           :isr       => [:int32])
+    TopicMetadataStruct = ProtocolStruct.new(:error => :int16,
+                                       :name => :string,
+                                       :partitions => [PartitionMetadata])
+    MetadataResponse = ProtocolStruct.new(:common => ResponseCommon,
+                                          :brokers => [Broker],
+                                          :topics => [TopicMetadata])
+  end
+end

data/lib/poseidon/sync_producer.rb

@@ -0,0 +1,117 @@
+module Poseidon
+  # Used by +Producer+ for sending messages to the kafka cluster.
+  #
+  # You should not use this interface directly
+  #
+  # Fetches metadata at appropriate times.
+  # Builds MessagesToSend
+  # Handle MessageBatchToSend lifecyle
+  #
+  # Who is responsible for fetching metadata from broker seed list?
+  #   Do we want to be fetching from real live brokers eventually?
+  #
+  # @api private
+  class SyncProducer
+    OPTION_DEFAULTS = {
+      :compression_codec => nil,
+      :compressed_topics => nil,
+      :metadata_refresh_interval_ms => 600_000,
+      :partitioner => nil,
+      :max_send_retries => 3,
+      :retry_backoff_ms => 100,
+      :required_acks => 0,
+      :ack_timeout_ms => 1500,
+    }
+
+    attr_reader :client_id, :retry_backoff_ms, :max_send_retries,
+      :metadata_refresh_interval_ms, :required_acks, :ack_timeout_ms
+    def initialize(client_id, seed_brokers, options = {})
+      @client_id = client_id
+
+      handle_options(options.dup)
+
+      @cluster_metadata   = ClusterMetadata.new
+      @message_conductor  = MessageConductor.new(@cluster_metadata, @partitioner)
+      @broker_pool        = BrokerPool.new(client_id, seed_brokers)
+    end
+
+    def send_messages(messages)
+      messages_to_send = MessagesToSend.new(messages, @cluster_metadata)
+
+      (@max_send_retries+1).times do
+        if messages_to_send.needs_metadata? || refresh_interval_elapsed?
+          refreshed_metadata = refresh_metadata(messages_to_send.topic_set)
+          if !refreshed_metadata
+            # If we can't refresh metadata we have to give up.
+            break
+          end
+        end
+
+        messages_to_send.messages_for_brokers(@message_conductor).each do |messages_for_broker|
+          if send_to_broker(messages_for_broker)
+            messages_to_send.successfully_sent(messages_for_broker)
+          end
+        end
+
+        if messages_to_send.all_sent? || @max_send_retries == 0
+          break
+        else
+          Kernel.sleep retry_backoff_ms / 1000.0
+          refresh_metadata(messages_to_send.topic_set)
+        end
+      end
+
+      messages_to_send.all_sent?
+    end
+
+    def shutdown
+      @broker_pool.shutdown
+    end
+    
+    private
+    def handle_options(options)
+      @ack_timeout_ms    = handle_option(options, :ack_timeout_ms)
+      @retry_backoff_ms  = handle_option(options, :retry_backoff_ms)
+
+      @metadata_refresh_interval_ms = 
+        handle_option(options, :metadata_refresh_interval_ms)
+
+      @required_acks    = handle_option(options, :required_acks)
+      @max_send_retries = handle_option(options, :max_send_retries)
+
+      @compression_config = ProducerCompressionConfig.new(
+        handle_option(options, :compression_codec), 
+        handle_option(options, :compressed_topics))
+
+      @partitioner = handle_option(options, :partitioner)
+
+      raise ArgumentError, "Unknown options: #{options.keys.inspect}" if options.keys.any?
+    end
+
+    def handle_option(options, sym)
+      options.delete(sym) || OPTION_DEFAULTS[sym]
+    end
+
+    def refresh_interval_elapsed?
+      (Time.now - @cluster_metadata.last_refreshed_at) > metadata_refresh_interval_ms
+    end
+
+    def refresh_metadata(topics)
+      @cluster_metadata.update(@broker_pool.fetch_metadata(topics))
+      @broker_pool.update_known_brokers(@cluster_metadata.brokers)
+      true
+    rescue Errors::UnableToFetchMetadata
+      false
+    end
+
+    def send_to_broker(messages_for_broker)
+      return false if messages_for_broker.broker_id == -1
+      to_send = messages_for_broker.build_protocol_objects(@compression_config)
+      @broker_pool.execute_api_call(messages_for_broker.broker_id, :produce, 
+                                    required_acks, ack_timeout_ms,
+                                    to_send)
+    rescue Connection::ConnectionFailedError
+      false
+    end
+  end
+end

data/lib/poseidon/topic_metadata.rb

@@ -0,0 +1,65 @@
+module Poseidon
+  # @api private
+  class TopicMetadata
+    # Build a new TopicMetadata object from its binary representation
+    #
+    # @param [ResponseBuffer] buffer
+    # @return [TopicMetadata]
+    #
+    def self.read(buffer)
+      tm = TopicMetadata.new
+      tm.struct = Protocol::TopicMetadataStruct.read(buffer)
+      tm
+    end
+
+    attr_accessor :struct
+    def initialize(struct=nil)
+      self.struct = struct
+    end
+
+    # Write a binary representation of the TopicMetadata to buffer
+    # 
+    # @param [RequestBuffer] buffer
+    # @return [nil]
+    def write(buffer)
+      struct.write(buffer)
+      nil
+    end
+
+    def partitions
+      struct.partitions
+    end
+
+    def name
+      struct.name
+    end
+
+    def ==(o)
+      eql?(o)
+    end
+
+    def eql?(o)
+      struct.eql?(o.struct)
+    end
+
+    def objects_with_errors
+      struct.objects_with_errors
+    end
+
+    def leader_available?
+      struct.error_class != Errors::LeaderNotAvailable
+    end
+
+    def partition_count
+      @partition_count ||= struct.partitions.count
+    end
+
+    def available_partition_leader_ids
+      @available_partition_leader_ids ||= struct.partitions.select(&:leader)
+    end
+
+    def available_partition_count
+      @available_partition_count ||= available_partition_leader_ids.count
+    end
+  end
+end

data/lib/poseidon/version.rb

@@ -0,0 +1,4 @@
+module Poseidon
+  # Unstable! API May Change!
+  VERSION = "0.0.1"
+end

data/lib/poseidon.rb

@@ -0,0 +1,102 @@
+# Stdlib requires
+require 'socket'
+require 'zlib'
+require 'thread'
+require 'set'
+
+# Public API
+require "poseidon/message_to_send"
+require "poseidon/producer"
+require "poseidon/fetched_message"
+require "poseidon/partition_consumer"
+
+# Poseidon!
+require "poseidon/message"
+require "poseidon/message_set"
+require "poseidon/topic_metadata"
+require "poseidon/protocol"
+
+require "poseidon/broker_pool"
+require "poseidon/cluster_metadata"
+require "poseidon/compression"
+require "poseidon/connection"
+require "poseidon/message_conductor"
+require "poseidon/messages_for_broker"
+require "poseidon/messages_to_send"
+require "poseidon/messages_to_send_batch"
+require "poseidon/producer_compression_config"
+require "poseidon/sync_producer"
+require "poseidon/version"
+
+# Top level Poseidon namespace
+#
+# @api public
+module Poseidon
+  # Posiedon exception namespace
+  module Errors
+    # @api private
+    class ProtocolError < StandardError; end
+
+    # Protocol Exceptions
+    #
+    # These are defined by the Poseidon wire format,
+    # they should be caught before being raised to users.
+    #
+    # @api private
+    class UnknownError < ProtocolError; end
+    # @api private
+    class OffsetOutOfRange < ProtocolError; end
+    # @api private
+    class InvalidMessage < ProtocolError; end
+    # @api private
+    class UnknownTopicOrPartition < ProtocolError; end
+    # @api private
+    class InvalidMessageSize < ProtocolError; end
+    # @api private
+    class LeaderNotAvailable < ProtocolError; end
+    # @api private
+    class NotLeaderForPartition < ProtocolError; end
+    # @api private
+    class RequestTimedOut < ProtocolError; end
+    # @api private
+    class BrokerNotAvailable < ProtocolError; end
+    # @api private
+    class ReplicaNotAvailable < ProtocolError; end
+    # @api private
+    class MessageSizeTooLarge < ProtocolError; end
+    # @api private
+    class UnrecognizedProtocolError < ProtocolError; end
+
+    # @api private
+    NO_ERROR_CODE = 0
+    # @api private
+    ERROR_CODES = {
+      -1 => UnknownError,
+      1 => OffsetOutOfRange,
+      2 => InvalidMessage,
+      3 => UnknownTopicOrPartition,
+      4 => InvalidMessageSize,
+      5 => LeaderNotAvailable,
+      6 => NotLeaderForPartition,
+      7 => RequestTimedOut,
+      8 => BrokerNotAvailable,
+      9 => ReplicaNotAvailable,
+      10 => MessageSizeTooLarge
+    }
+
+    # Raised when a custom partitioner tries to send
+    # a message to a partition that doesn't exist.
+    class InvalidPartitionError < StandardError; end
+
+    # Raised when we are unable to fetch metadata from
+    # any of the brokers.
+    class UnableToFetchMetadata < StandardError; end
+
+    # Raised when a messages checksum doesn't match
+    class ChecksumError < StandardError; end
+
+    # Raised when you try to send messages to a producer
+    # object that has been #shutdown
+    class ProducerShutdownError < StandardError; end
+  end
+end

data/poseidon.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'poseidon/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "poseidon"
+  gem.version       = Poseidon::VERSION
+  gem.authors       = ["Bob Potter"]
+  gem.email         = ["bobby.potter@gmail.com"]
+  gem.description   = %q{A Kafka (http://kafka.apache.org/) producer and consumer}
+  gem.summary       = %q{Poseidon is a producer and consumer implementation for Kafka >= 0.8}
+  gem.homepage      = "https://github.com/bpot/poseidon"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_development_dependency(%q<rspec>)
+  gem.add_development_dependency(%q<yard>)
+  gem.add_development_dependency(%q<simplecov>)
+  gem.add_development_dependency(%q<daemon_controller>)
+end

data/spec/bin/kafka-run-class.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# 
+#    http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ];
+then
+  echo "USAGE: $0 classname [opts]"
+  exit 1
+fi
+
+SCALA_VERSION=2.8.0
+
+# assume all dependencies have been packaged into one jar with sbt-assembly's task "assembly-package-dependency"
+for file in $KAFKA_PATH/core/target/scala-2.8.0/*.jar;
+do
+  CLASSPATH=$CLASSPATH:$file
+done
+
+for file in $KAFKA_PATH/perf/target/scala-${SCALA_VERSION}/kafka*.jar;
+do
+  CLASSPATH=$CLASSPATH:$file
+done
+
+# classpath addition for release
+for file in $KAFKA_PATH/libs/*.jar;
+do
+  CLASSPATH=$CLASSPATH:$file
+done
+
+for file in $KAFKA_PATH/kafka*.jar;
+do
+  CLASSPATH=$CLASSPATH:$file
+done
+
+if [ -z "$KAFKA_JMX_OPTS" ]; then
+  KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false  -Dcom.sun.management.jmxremote.ssl=false "
+fi
+
+if [ -z "$KAFKA_OPTS" ]; then
+  KAFKA_OPTS="-Xmx512M -server  -Dlog4j.configuration=file:$KAFKA_PATH/config/log4j.properties"
+fi
+
+if [  $JMX_PORT ]; then
+  KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.port=$JMX_PORT "
+fi
+
+if [ -z "$JAVA_HOME" ]; then
+  JAVA="java"
+else
+  JAVA="$JAVA_HOME/bin/java"
+fi
+
+exec $JAVA $KAFKA_OPTS $KAFKA_JMX_OPTS -cp $CLASSPATH "$@"

data/spec/integration/multiple_brokers/round_robin_spec.rb

@@ -0,0 +1,39 @@
+require 'integration/multiple_brokers/spec_helper'
+
+describe "round robin sending" do
+  describe "with small message batches" do
+    it "evenly distributes messages across brokers" do
+      c = Connection.new("localhost", 9092, "metadata_fetcher")
+      md = c.topic_metadata(["test"])
+      sleep 1
+      md = c.topic_metadata(["test"])
+
+      test_topic = md.topics.first
+
+      consumers = test_topic.partitions.map do |partition|
+        leader_id = partition.leader
+        broker = md.brokers.find { |b| b.id == leader_id }
+        PartitionConsumer.new("test_consumer_#{partition.id}", broker.host,
+                              broker.port, "test", partition.id, -1)
+      end
+
+      # Update offsets to current position before adding test messages
+      consumers.each do |c|
+        c.fetch
+      end
+
+
+      @p = Producer.new(["localhost:9092","localhost:9093","localhost:9094"], "test",
+                       :required_acks => 1)
+
+      24.times do
+        @p.send_messages([MessageToSend.new("test", "hello")])
+      end
+
+      consumers.each do |c|
+        messages = c.fetch
+        expect(messages.size).to eq(8)
+      end
+    end
+  end
+end

data/spec/integration/multiple_brokers/spec_helper.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+require 'test_cluster'
+
+class ThreeBrokerCluster
+  def initialize
+    @zookeeper = ZookeeperRunner.new
+    @brokers = (9092..9094).map { |port| BrokerRunner.new(port - 9092, port, 3) }
+  end
+
+  def start
+    @zookeeper.start
+    @brokers.each(&:start)
+  end
+
+  def stop
+    @zookeeper.stop
+    @brokers.each(&:stop)
+  end
+end
+
+RSpec.configure do |config|
+  config.before(:suite) do
+    JavaRunner.remove_tmp
+    JavaRunner.set_kafka_path!
+    $tc = ThreeBrokerCluster.new
+    $tc.start
+    sleep 5 # wait for cluster to come up
+  end
+
+  config.after(:suite) do
+    $tc.stop
+  end
+end

data/spec/integration/simple/compression_spec.rb

@@ -0,0 +1,20 @@
+require 'integration/simple/spec_helper'
+
+describe "compression" do
+  it "roundtrips" do
+    i = rand(1000)
+
+    @consumer = PartitionConsumer.new("test_consumer", "localhost", 9092,
+                                      "test12", 0, -2)
+
+    @producer = Producer.new(["localhost:9092"],
+                             "test_client",
+                             :type => :sync,
+                             :compression_codec => :gzip)
+    messages = [MessageToSend.new("test12", "Hello World: #{i}")]
+    
+    expect(@producer.send_messages(messages)).to eq(true)
+    messages = @consumer.fetch
+    expect(messages.last.value).to eq("Hello World: #{i}")
+  end
+end

data/spec/integration/simple/connection_spec.rb

@@ -0,0 +1,33 @@
+require 'integration/simple/spec_helper'
+
+include Protocol
+describe Connection do
+  before(:each) do
+    @connection = Connection.new("localhost", 9092, "test")
+  end
+
+  it 'sends and parses topic metadata requests' do
+    @connection.topic_metadata(["test2"])
+  end
+
+  it 'sends and parsers produce requests' do
+    message = MessageStruct.new(0, 0, nil, "hello") 
+    message_with_offset = MessageWithOffsetStruct.new(0, message)
+    message_set = MessageSetStruct.new([message_with_offset])
+    messages_for_partitions = [MessagesForPartition.new(0,message_set)] 
+    messages_for_topics = [MessagesForTopic.new("test2",messages_for_partitions)]
+    @connection.produce(1, 10_000, messages_for_topics)
+  end
+
+  it 'sends and parsers fetch requests' do
+    partition_fetches = [PartitionFetch.new(0,0,1024*1024)]
+    topic_fetches = [TopicFetch.new("test2", partition_fetches)]
+    @connection.fetch(1000, 0, topic_fetches)
+  end
+  
+  it 'sends and parsers offset requests' do
+    partition_offset_requests = [PartitionOffsetRequest.new(0,-1,1000)]
+    offset_topic_requests = [TopicOffsetRequest.new("test2", partition_offset_requests)]
+    @connection.offset(offset_topic_requests)
+  end
+end

data/spec/integration/simple/multiple_brokers_spec.rb

@@ -0,0 +1,8 @@
+require 'integration/simple/spec_helper'
+
+describe "three brokers in cluster" do
+  describe "sending batches of 1 message" do
+    it "sends messages to all brokers" do
+    end
+  end
+end