heller 0.0.1-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,10 @@
1
+ # encoding: utf-8
2
+
3
+ require 'kafka'
4
+
5
+ module Heller
6
+ java_import 'java.util.ArrayList'
7
+ end
8
+
9
+ require 'heller/consumer'
10
+ require 'heller/producer'
@@ -0,0 +1,53 @@
1
+ module Heller
2
+ class Consumer < Kafka::Consumer::SimpleConsumer
3
+
4
+ LATEST_OFFSET = -1.freeze
5
+ EARLIEST_OFFSET = -2.freeze
6
+
7
+ MAX_FETCH_SIZE = 1000000.freeze
8
+
9
+ def latest_offset(topic, partition)
10
+ single_offset(topic, partition, LATEST_OFFSET)
11
+ end
12
+
13
+ def earliest_offset(topic, partition)
14
+ single_offset(topic, partition, EARLIEST_OFFSET)
15
+ end
16
+
17
+ def fetch_request(topic, partition, offset, max_size)
18
+ Kafka::Api::FetchRequest.new(topic, partition, offset, max_size)
19
+ end
20
+
21
+ def consume(topic, partition, offset, max_size = MAX_FETCH_SIZE)
22
+ request = fetch_request(topic, partition, offset, max_size)
23
+
24
+ messages = fetch(request)
25
+ messages.to_a
26
+ end
27
+
28
+ def multi_fetch(topics_hash, max_size = MAX_FETCH_SIZE)
29
+ requests = topics_hash.map { |topic, hash| fetch_request(topic, hash[:partition], hash[:offset], max_size) }
30
+
31
+ response = multifetch(ArrayList.new(requests))
32
+ parse_multi_fetch_response(topics_hash.keys, response)
33
+ end
34
+
35
+ protected
36
+
37
+ def single_offset(topic, partition, time)
38
+ offsets = get_offsets_before(topic, partition, time, 1)
39
+ offsets = offsets.to_a
40
+
41
+ offsets.first unless offsets.empty?
42
+ end
43
+
44
+ def parse_multi_fetch_response(topics, response)
45
+ response_array = topics.zip(response.to_a)
46
+
47
+ response_array.inject({}) do |response_hash, (topic, messages)|
48
+ response_hash[topic] = messages.to_a
49
+ response_hash
50
+ end
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,48 @@
1
+ module Heller
2
+ class Producer < Kafka::Producer::SyncProducer
3
+
4
+ attr_reader :configuration
5
+
6
+ def initialize(host, port, options = {})
7
+ options.merge!({
8
+ 'host' => host,
9
+ 'port' => port.to_s
10
+ })
11
+
12
+ @configuration = Kafka::Producer::SyncProducerConfig.new(hash_to_properties(options))
13
+
14
+ super(@configuration)
15
+ end
16
+
17
+ def wrap_messages(messages)
18
+ converted = messages.map do |m|
19
+ Kafka::Message::Message.new(m)
20
+ end
21
+
22
+ Kafka::Message::ByteBufferMessageSet.new(ArrayList.new(converted))
23
+ end
24
+
25
+ def produce(topic, messages, partition = :random)
26
+ message_set = wrap_messages(messages)
27
+
28
+ case partition
29
+ when :random
30
+ send(topic, message_set)
31
+ when Integer
32
+ send(topic, partition, message_set)
33
+ end
34
+ end
35
+
36
+ protected
37
+
38
+ def hash_to_properties(options)
39
+ properties = java.util.Properties.new
40
+
41
+ options.each do |key, value|
42
+ properties.put(key.to_s, value.to_s)
43
+ end
44
+
45
+ properties
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,27 @@
1
+ # encoding: utf-8
2
+
3
+ require 'java'
4
+ require 'kafka-jars'
5
+
6
+ module Kafka
7
+ module Api
8
+ java_import 'kafka.api.FetchRequest'
9
+ java_import 'kafka.api.MultiFetchRequest'
10
+ java_import 'kafka.api.MultiFetchResponse'
11
+ end
12
+
13
+ module Consumer
14
+ java_import 'kafka.javaapi.consumer.SimpleConsumer'
15
+ end
16
+
17
+ module Message
18
+ java_import 'kafka.message.Message'
19
+ java_import 'kafka.javaapi.message.MessageSet'
20
+ java_import 'kafka.javaapi.message.ByteBufferMessageSet'
21
+ end
22
+
23
+ module Producer
24
+ java_import 'kafka.producer.SyncProducerConfig'
25
+ java_import 'kafka.javaapi.producer.SyncProducer'
26
+ end
27
+ end
metadata ADDED
@@ -0,0 +1,71 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: heller
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ prerelease:
6
+ platform: java
7
+ authors:
8
+ - Mathias Söderberg
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2012-10-23 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: kafka-jars
16
+ version_requirements: !ruby/object:Gem::Requirement
17
+ requirements:
18
+ - - ~>
19
+ - !ruby/object:Gem::Version
20
+ version: 0.7.2.1
21
+ none: false
22
+ requirement: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ~>
25
+ - !ruby/object:Gem::Version
26
+ version: 0.7.2.1
27
+ none: false
28
+ prerelease: false
29
+ type: :runtime
30
+ description: Attempts to make Kafka's Java API fit a bit better with Ruby
31
+ email:
32
+ - mathias.soederberg@gmail.com
33
+ executables: []
34
+ extensions: []
35
+ extra_rdoc_files: []
36
+ files:
37
+ - !binary |-
38
+ bGliL2hlbGxlci5yYg==
39
+ - !binary |-
40
+ bGliL2thZmthLnJi
41
+ - !binary |-
42
+ bGliL2hlbGxlci9jb25zdW1lci5yYg==
43
+ - !binary |-
44
+ bGliL2hlbGxlci9wcm9kdWNlci5yYg==
45
+ homepage: http://github.com/mthssdrbrg/heller
46
+ licenses: []
47
+ post_install_message:
48
+ rdoc_options: []
49
+ require_paths:
50
+ - lib
51
+ required_ruby_version: !ruby/object:Gem::Requirement
52
+ requirements:
53
+ - - ! '>='
54
+ - !ruby/object:Gem::Version
55
+ version: !binary |-
56
+ MA==
57
+ none: false
58
+ required_rubygems_version: !ruby/object:Gem::Requirement
59
+ requirements:
60
+ - - ! '>='
61
+ - !ruby/object:Gem::Version
62
+ version: !binary |-
63
+ MA==
64
+ none: false
65
+ requirements: []
66
+ rubyforge_project: heller
67
+ rubygems_version: 1.8.24
68
+ signing_key:
69
+ specification_version: 3
70
+ summary: JRuby wrapper for Kafka
71
+ test_files: []