heller 0.0.1-java

data/lib/heller.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+
+require 'kafka'
+
+module Heller
+	java_import 'java.util.ArrayList'
+end
+
+require 'heller/consumer'
+require 'heller/producer'

data/lib/heller/consumer.rb

@@ -0,0 +1,53 @@
+module Heller
+	class Consumer < Kafka::Consumer::SimpleConsumer
+				
+		LATEST_OFFSET = -1.freeze
+		EARLIEST_OFFSET = -2.freeze
+
+		MAX_FETCH_SIZE = 1000000.freeze
+
+		def latest_offset(topic, partition)
+			single_offset(topic, partition, LATEST_OFFSET)
+		end
+
+		def earliest_offset(topic, partition)
+			single_offset(topic, partition, EARLIEST_OFFSET)
+		end
+
+		def fetch_request(topic, partition, offset, max_size)
+			Kafka::Api::FetchRequest.new(topic, partition, offset, max_size)
+		end
+
+		def consume(topic, partition, offset, max_size = MAX_FETCH_SIZE)
+			request = fetch_request(topic, partition, offset, max_size)
+
+			messages = fetch(request)
+			messages.to_a
+		end
+
+		def multi_fetch(topics_hash, max_size = MAX_FETCH_SIZE)
+			requests = topics_hash.map { |topic, hash| fetch_request(topic, hash[:partition], hash[:offset], max_size) }
+
+			response = multifetch(ArrayList.new(requests))
+			parse_multi_fetch_response(topics_hash.keys, response)
+		end
+
+		protected
+
+		def single_offset(topic, partition, time)
+			offsets = get_offsets_before(topic, partition, time, 1)
+			offsets = offsets.to_a
+
+			offsets.first unless offsets.empty?
+		end
+
+		def parse_multi_fetch_response(topics, response)
+			response_array = topics.zip(response.to_a)
+
+			response_array.inject({}) do |response_hash, (topic, messages)| 
+				response_hash[topic] = messages.to_a
+				response_hash
+			end
+		end
+	end
+end

data/lib/heller/producer.rb

@@ -0,0 +1,48 @@
+module Heller
+	class Producer < Kafka::Producer::SyncProducer
+
+		attr_reader :configuration
+
+		def initialize(host, port, options = {})
+			options.merge!({
+				'host' => host,
+				'port' => port.to_s	
+			})
+
+			@configuration = Kafka::Producer::SyncProducerConfig.new(hash_to_properties(options))
+
+			super(@configuration)
+		end
+
+		def wrap_messages(messages)
+			converted = messages.map do |m| 
+				Kafka::Message::Message.new(m) 
+			end
+
+			Kafka::Message::ByteBufferMessageSet.new(ArrayList.new(converted))
+		end
+
+		def produce(topic, messages, partition = :random)
+			message_set = wrap_messages(messages)
+
+			case partition
+			when :random
+				send(topic, message_set)
+			when Integer
+				send(topic, partition, message_set)
+			end
+		end
+
+		protected
+
+		def hash_to_properties(options)
+			properties = java.util.Properties.new
+
+			options.each do |key, value|
+				properties.put(key.to_s, value.to_s)
+			end
+
+			properties
+		end
+	end
+end

data/lib/kafka.rb

@@ -0,0 +1,27 @@
+# encoding: utf-8
+
+require 'java'
+require 'kafka-jars'
+
+module Kafka
+	module Api
+		java_import 'kafka.api.FetchRequest'
+		java_import 'kafka.api.MultiFetchRequest'
+		java_import 'kafka.api.MultiFetchResponse'
+	end
+
+	module Consumer
+		java_import 'kafka.javaapi.consumer.SimpleConsumer'
+	end
+
+	module Message
+		java_import 'kafka.message.Message'
+		java_import 'kafka.javaapi.message.MessageSet'
+		java_import 'kafka.javaapi.message.ByteBufferMessageSet'
+	end
+
+	module Producer
+		java_import 'kafka.producer.SyncProducerConfig'
+		java_import 'kafka.javaapi.producer.SyncProducer'
+	end
+end

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification
+name: heller
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease:
+platform: java
+authors:
+- Mathias Söderberg
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2012-10-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: kafka-jars
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.2.1
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.2.1
+    none: false
+  prerelease: false
+  type: :runtime
+description: Attempts to make Kafka's Java API fit a bit better with Ruby
+email:
+- mathias.soederberg@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- !binary |-
+  bGliL2hlbGxlci5yYg==
+- !binary |-
+  bGliL2thZmthLnJi
+- !binary |-
+  bGliL2hlbGxlci9jb25zdW1lci5yYg==
+- !binary |-
+  bGliL2hlbGxlci9wcm9kdWNlci5yYg==
+homepage: http://github.com/mthssdrbrg/heller
+licenses: []
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: !binary |-
+        MA==
+  none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: !binary |-
+        MA==
+  none: false
+requirements: []
+rubyforge_project: heller
+rubygems_version: 1.8.24
+signing_key:
+specification_version: 3
+summary: JRuby wrapper for Kafka
+test_files: []