poseidon 0.0.1

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.log
+*.log.*

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/.travis.yml

@@ -0,0 +1,12 @@
+laguage: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - ruby-head
+  - jruby-19mode
+  - jruby-head
+  - rbx-19mode
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+    - rvm: rbx-19mode

data/.yardopts

@@ -0,0 +1,8 @@
+--markup-provider=redcarpet
+--markup=markdown
+--no-private
+--files LICENSE.txt
+--exclude '~$'
+--title 'Poseidon (Kafka library for Ruby)'
+--api public
+--readme README.md

data/Gemfile

@@ -0,0 +1,13 @@
+source 'https://rubygems.org'
+
+gem 'rake'
+
+# Specify your gem's dependencies in poseidon.gemspec
+gemspec
+
+gem 'coveralls', require: false
+
+group :development do
+  gem 'github-markup', :platform => :ruby
+  gem 'redcarpet', :platform => :ruby
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Bob Potter
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,71 @@
+# Poseidon [![Build Status](https://travis-ci.org/bpot/poseidon.png?branch=master)](https://travis-ci.org/bpot/poseidon) [![Code Climate](https://codeclimate.com/github/bpot/poseidon.png)](https://codeclimate.com/github/bpot/poseidon)
+
+Poseidon is a Kafka client. Poseidon only supports the 0.8 API and above.
+
+**Until 1.0.0 this should be considered ALPHA software and not neccessarily production ready.**
+
+* [API Documentation](http://rubydoc.info/github/bpot/poseidon)
+
+## Usage
+
+### Installing a Kafka broker locally
+
+Follow the [instructions](https://cwiki.apache.org/KAFKA/kafka-08-quick-start.html) on the Kafka wiki to build Kafka 0.8 and get a test broker up and running.
+
+### Sending messages to Kafka
+
+```ruby
+require 'poseidon'
+
+producer = Poseidon::Producer.new(["localhost:9092"], "my_test_producer")
+
+messages = []
+messages << Poseidon::MessageToSend.new("topic1", "value1")
+messages << Poseidon::MessageToSend.new("topic2", "value2")
+producer.send_messages(messages)
+```
+
+More detailed [Poseidon::Producer](http://rubydoc.info/github/bpot/poseidon/Poseidon/Producer) documentation.
+
+
+### Fetching messages from Kafka
+
+```ruby
+require 'poseidon'
+
+consumer = Poseidon::PartitionConsumer.new("my_test_consumer", "localhost", 9092,
+                                            "topic1", 0, :earliest_offset)
+
+loop do
+  messages = consumer.fetch
+  messages.each do |m|
+    puts m.value
+  end
+end
+```
+
+More detailed [Poseidon::PartitionConsumer](http://rubydoc.info/github/bpot/poseidon/Poseidon/PartitionConsumer) documentation.
+
+## Semantic Versioning
+
+This gem follows [SemVer](http://semver.org). In particular, the public API should not be considered stable and anything may change without warning until Version 1.0.0.  Additionally, for the purposes of the versioning the public API is everything documented in the [public API docs](http://rubydoc.info/github/bpot/poseidon).
+
+## Requirements
+
+* Ruby 1.9.3 or higher (1.9.2 and below not supported!!!)
+* Kafka 0.8 or higher
+
+## Integration Tests
+
+In order to run integration tests you must specify a `KAFKA_PATH` environment variable which points to a built Kafka installation.  There are more detailed [instructions](https://cwiki.apache.org/KAFKA/kafka-08-quick-start.html) on the Kafka wiki, but the following should allow you to run integration tests.
+
+    # cd ~/src/
+    # git clone https://git-wip-us.apache.org/repos/asf/kafka.git
+    # git checkout -b 0.8 remotes/origin/0.8
+    # ./sbt update
+    # ./sbt package
+    # ./sbt assembly-package-dependency
+    # cd ~/src/poseidon/
+    # KAFKA_PATH=~/src/kafka rake spec:integration:simple
+
+The poseidon test suite will take care of spinning up and down the broker(s) needed for the integration tests.

data/Rakefile

@@ -0,0 +1,17 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new("spec:unit") do |t|
+  t.pattern = 'spec/unit/*_spec.rb'
+end
+
+RSpec::Core::RakeTask.new('spec:integration:simple') do |t|
+  t.pattern = 'spec/integration/simple/*_spec.rb'
+end
+
+RSpec::Core::RakeTask.new('spec:integration:multiple_brokers') do |t|
+  t.pattern = 'spec/integration/multiple_brokers/*_spec.rb'
+end
+
+task :spec => 'spec:unit'
+task :default => 'spec:unit'

data/TODO.md

@@ -0,0 +1,27 @@
+### 0.0.1
+  * Ensure that protocol errors are being handled correctly and not bubbling up
+  * More integration tests, replication, leader changes, etc. Investigate interesting cases in kafka's tests
+  * End-to-end integration specs
+    - In specs that test broker failure, verify that messages were actually sent/not sent with a consumer.
+ 
+  * AsyncProducer
+    - Implement a bounded queue, sending thread, etc
+  * Cleanup: extract protocol struct delegation to a module.
+  * When failing to send messages in sync producer, return messages that failed to send?
+
+### 0.0.2
+
+  * New Consumer/Consumer Enhancements
+    - Automatically partition work among consumers (zookeeper, redis, pluggable?)
+    - Handle case where the offset we're trying to read from no longer exists
+
+  * Snappy Compression
+    - snappy: c-ext, would like to avoid
+    - snappy_ffi: ffi interface, but needs to be updated (pre c-api) 
+        and has no specs, docs. Also linked to a c-ext version, two gems, etc..
+    - new snappy ffi library with specs, docs, etc. Shave that Yak!
+
+  * Benchmark/Profiling. KGIO?
+
+### 0.0.3 -- Targets Kafka 0.8.1
+  - Offset API

data/examples/consumer.rb

@@ -0,0 +1,18 @@
+$:.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'poseidon'
+
+producer = Poseidon::PartitionConsumer.new("example_consumer", "localhost", 9092,
+                                           "example", 0, :earliest_offset)
+
+loop do
+  begin
+    messages = producer.fetch
+    messages.each do |m|
+      puts "Received message: #{m.value}"
+    end
+  rescue Poseidon::Errors::UnknownTopicOrPartition
+    puts "Topic does not exist yet"
+  end
+
+  sleep 1
+end

data/examples/producer.rb

@@ -0,0 +1,9 @@
+$:.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'poseidon'
+
+producer = Poseidon::Producer.new(["localhost:9092"], "example_producer")
+
+loop do
+  producer.send_messages([Poseidon::MessageToSend.new("example", Time.now.to_s)])
+  sleep 1
+end

data/lib/poseidon/broker_pool.rb

@@ -0,0 +1,72 @@
+module Poseidon
+  # BrokerPool allows you to send api calls to the a brokers Connection.
+  # 
+  # @api private
+  class BrokerPool
+    class UnknownBroker < StandardError; end
+
+    # @param [String] client_id
+    def initialize(client_id, seed_brokers)
+      @connections = {}
+      @brokers = {}
+      @client_id = client_id
+      @seed_brokers = seed_brokers
+    end
+
+    def fetch_metadata(topics)
+      @seed_brokers.each do |broker|
+        if metadata = fetch_metadata_from_broker(broker, topics)
+          return metadata
+        end
+      end
+      raise Errors::UnableToFetchMetadata
+    end
+
+    # Update the brokers we know about
+    #
+    # TODO break connection when a brokers info changes?
+    #
+    # @param [Hash<Integer,Hash>] brokers
+    #   Hash of broker_id => { :host => host, :port => port }
+    def update_known_brokers(brokers)
+      @brokers.update(brokers)
+      nil
+    end
+
+    # Executes an api call on the connection
+    #
+    # @param [Integer] broker_id id of the broker we want to execute it on
+    # @param [Symbol] api_call
+    #   the api call we want to execute (:produce,:fetch,etc)
+    def execute_api_call(broker_id, api_call, *args)
+      connection(broker_id).send(api_call, *args)
+    end
+
+    # Closes all open connections to brokers
+    def shutdown
+      @brokers.values(&:close)
+      @brokers = {}
+    end
+
+    private
+    def fetch_metadata_from_broker(broker, topics)
+      host, port = broker.split(":")
+      c = Connection.new(host, port, @client_id)
+      c.topic_metadata(topics)
+    rescue Connection::ConnectionFailedError
+      return nil
+    end
+
+    def connection(broker_id)
+      @connections[broker_id] ||= new_connection(broker_id)
+    end
+
+    def new_connection(broker_id)
+      info = @brokers[broker_id]
+      if info.nil?
+        raise UnknownBroker
+      end
+      Connection.new(info[:host], info[:port], @client_id)
+    end
+  end
+end

data/lib/poseidon/cluster_metadata.rb

@@ -0,0 +1,63 @@
+module Poseidon
+  # Encapsulates what we known about brokers, topics and partitions
+  # from Metadata API calls.
+  #
+  # @api private
+  class ClusterMetadata
+    attr_reader :brokers, :last_refreshed_at, :topic_metadata
+    def initialize
+      @brokers        = {}
+      @topic_metadata = {}
+      @last_refreshed_at = nil
+    end
+
+    # Update what we know about the cluter based on MetadataResponse
+    # 
+    # @param [MetadataResponse] topic_metadata_response
+    # @return nil
+    def update(topic_metadata_response)
+      update_brokers(topic_metadata_response.brokers)
+      update_topics(topic_metadata_response.topics)
+
+      @last_refreshed_at = Time.now
+      nil
+    end
+
+    # Do we have metadata for these topics already?
+    #
+    # @param [Enumberable<String>] topic_names A set of topics.
+    # @return [Boolean] true if we have metadata for all +topic_names+, otherwise false.
+    def have_metadata_for_topics?(topic_names)
+      topic_names.all? { |topic| @topic_metadata[topic] }
+    end
+
+    # Provides metadata for each topic
+    #
+    # @param [Enumerable<String>] topic_names Topics we should return metadata for
+    # @return [Hash<String,TopicMetadata>]
+    def metadata_for_topics(topic_names)
+      Hash[topic_names.map { |name| [name, @topic_metadata[name]] }]
+    end
+
+    # Provides a Broker object for +broker_id+. This corresponds to the
+    # broker ids in the TopicMetadata objects.
+    #
+    # @param [Integer] broker_id Broker id 
+    def broker(broker_id)
+      @brokers[broker_id]
+    end
+
+    private
+    def update_topics(topics)
+      topics.each do |topic|
+        @topic_metadata[topic.name] = topic
+      end
+    end
+
+    def update_brokers(brokers)
+      brokers.each do |broker|
+        @brokers[broker.id] = broker
+      end
+    end
+  end
+end

data/lib/poseidon/compressed_value.rb

@@ -0,0 +1,23 @@
+module Poseidon
+  # @api private
+  class CompressedValue
+    def initialize(value, codec_id)
+      @value = value
+      @codec_id = codec_id
+    end
+
+    # Decompressed value
+    #
+    # Raises ??? if the compression codec is uknown
+    #
+    # @return [String] decompressed value
+    def decompressed
+      @decompressed ||= decompress
+    end
+
+    def compression_codec
+      Compression.find_codec(codec_id)
+    end
+    private
+  end
+end

data/lib/poseidon/compression/gzip_codec.rb

@@ -0,0 +1,23 @@
+module Poseidon
+  module Compression
+    module GzipCodec
+      def self.codec_id
+        1
+      end
+
+      def self.compress(s)
+        io = StringIO.new
+        io.set_encoding("ASCII-8BIT")
+        gz = Zlib::GzipWriter.new io, Zlib::DEFAULT_COMPRESSION, Zlib::DEFAULT_STRATEGY, :encoding => "ASCII-8BIT"
+        gz.write s
+        gz.close
+        io.string
+      end
+
+      def self.decompress(s)
+        io = StringIO.new(s)
+        Zlib::GzipReader.new(io, :encoding => "ASCII-8BIT").read
+      end
+    end
+  end
+end

data/lib/poseidon/compression/snappy_codec.rb

@@ -0,0 +1,17 @@
+module Poseidon
+  module Compression
+    module SnappyCodec
+      def self.codec_id
+        2
+      end
+
+      def self.compress(s)
+        raise "Unimplemented"
+      end
+
+      def self.decompress(s)
+        raise "Unimplemented"
+      end
+    end
+  end
+end

data/lib/poseidon/compression.rb

@@ -0,0 +1,30 @@
+module Poseidon
+  # @api private
+  module Compression
+    class UnrecognizedCompressionCodec < StandardError; end
+
+    require "poseidon/compression/gzip_codec"
+    require "poseidon/compression/snappy_codec"
+
+    CODECS = {
+      #0 => no codec
+      1 => GzipCodec,
+      2 => SnappyCodec
+    }
+
+    # Fetches codec module for +codec_id+
+    # https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Compression
+    #
+    # @param [Integer] codec_id codec's as defined by the Kafka Protocol
+    # @return [Module] codec module for codec_id
+    #
+    # @private
+    def self.find_codec(codec_id)
+      codec = CODECS[codec_id]
+      if codec.nil?
+        raise UnrecognizedCompressionCodec, codec_id
+      end
+      codec
+    end
+  end
+end

data/lib/poseidon/connection.rb

@@ -0,0 +1,138 @@
+module Poseidon
+  # High level internal interface to a remote broker. Provides access to
+  # the broker API.
+  # @api private
+  class Connection
+    include Protocol
+
+    class ConnectionFailedError < StandardError; end
+
+    API_VERSION = 0
+    REPLICA_ID = -1 # Replica id is always -1 for non-brokers
+
+    attr_reader :host, :port
+
+    # Create a new connection
+    #
+    # @param [String] host Host to connect to
+    # @param [Integer] port Port broker listens on
+    # @param [String] client_id Unique across processes?
+    def initialize(host, port, client_id)
+      @host = host
+      @port = port
+
+      @client_id = client_id
+    end
+
+    # Close broker connection
+    def close
+      @socket && @socket.close
+    end
+  
+    # Execute a produce call
+    #
+    # @param [Integer] required_acks
+    # @param [Integer] timeout
+    # @param [Array<Protocol::MessagesForTopics>] messages_for_topics Messages to send
+    # @return [ProduceResponse]
+    def produce(required_acks, timeout, messages_for_topics)
+      ensure_connected
+      req = ProduceRequest.new( request_common(:produce),
+                                required_acks,
+                                timeout,
+                                messages_for_topics) 
+      send_request(req)
+      if required_acks != 0
+        read_response(ProduceResponse)
+      else
+        true
+      end
+    end
+
+    # Execute a fetch call
+    #
+    # @param [Integer] max_wait_time
+    # @param [Integer] min_bytes
+    # @param [Integer] topic_fetches
+    def fetch(max_wait_time, min_bytes, topic_fetches)
+      ensure_connected
+      req = FetchRequest.new( request_common(:fetch),
+                                REPLICA_ID,
+                                max_wait_time,
+                                min_bytes,
+                                topic_fetches) 
+      send_request(req)
+      read_response(FetchResponse)
+    end
+
+    def offset(offset_topic_requests)
+      ensure_connected
+      req = OffsetRequest.new(request_common(:offset),
+                              REPLICA_ID,
+                              offset_topic_requests)
+      send_request(req)
+      read_response(OffsetResponse).topic_offset_responses
+    end
+
+    # Fetch metadata for +topic_names+
+    #
+    # @param [Enumberable<String>] topic_names
+    #   A list of topics to retrive metadata for
+    # @return [TopicMetadataResponse] metadata for the topics
+    def topic_metadata(topic_names)
+      ensure_connected
+      req = MetadataRequest.new( request_common(:metadata),
+                                 topic_names)
+      send_request(req)
+      read_response(MetadataResponse)
+    end
+
+    private
+    def ensure_connected
+      if @socket.nil? || @socket.closed?
+        begin
+          @socket = TCPSocket.new(@host, @port)
+        rescue SystemCallError
+          raise ConnectionFailedError
+        end
+      end
+    end
+
+    def read_response(response_class)
+      r = @socket.read(4)
+      if r.nil?
+        raise ConnectionFailedError
+      end
+      n = r.unpack("N").first
+      s = @socket.read(n)
+      buffer = Protocol::ResponseBuffer.new(s)
+      response_class.read(buffer)
+    rescue Errno::ECONNRESET
+      @socket = nil
+      raise ConnectionFailedError
+    end
+
+    def send_request(request)
+      buffer = Protocol::RequestBuffer.new
+      request.write(buffer)
+      @socket.write([buffer.to_s.size].pack("N") + buffer.to_s)
+    rescue Errno::EPIPE
+      @socket = nil
+      raise ConnectionFailedError
+    end
+
+    def request_common(request_type)
+      RequestCommon.new(
+        API_KEYS[request_type],
+        API_VERSION,
+        next_correlation_id,
+        @client_id
+      )
+    end
+
+    def next_correlation_id
+      @correlation_id ||= 0
+      @correlation_id  += 1
+    end
+  end
+end

data/lib/poseidon/fetched_message.rb

@@ -0,0 +1,37 @@
+module Poseidon
+
+  # A message fetched from a Kafka broker.
+  #
+  # ```
+  # fetched_messages = consumer.fetch
+  # fetched_messages.each do |fm|
+  #   puts "Topic: #{fm.topic}"
+  #   puts "Value #{fm.value}"
+  #   puts "Key: #{fm.key}"
+  #   puts "Offset: #{fm.offset}"
+  # end
+  # ```
+  #
+  # @param [String] topic
+  #   Topic this message should be sent to.
+  #
+  # @param [String] value
+  #   Value of the message we want to send. 
+  #
+  # @param [String] key
+  #   Optional. Message's key, used to route a message
+  #   to a specific broker.  Otherwise, keys will be
+  #   sent to brokers in a round-robin manner.
+  #
+  # @api public
+  class FetchedMessage
+    attr_reader :value, :key, :topic, :offset
+
+    def initialize(topic, value, key, offset)
+      @topic  = topic
+      @value  = value
+      @key    = key
+      @offset = offset
+    end
+  end
+end