hermann 0.18.1-java → 0.19.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ext/hermann/hermann_lib.c +4 -5
- data/lib/hermann.rb +27 -12
- data/lib/hermann/consumer.rb +38 -5
- data/lib/hermann/errors.rb +14 -1
- data/lib/hermann/java.rb +20 -0
- data/lib/hermann/producer.rb +10 -7
- data/lib/hermann/provider/java_producer.rb +21 -34
- data/lib/hermann/provider/java_simple_consumer.rb +122 -0
- data/lib/hermann/version.rb +1 -1
- metadata +18 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a6ae048936233d039e0422d5fcf03deff436b95a
|
4
|
+
data.tar.gz: c9d8807df3dd9d8e9efd089a325471b24a27bb3f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c38aeb5099c6cf1f33b262b3a9f2b552ab964fc35d3c19a48280005580cf8d6b74401e15352ad93f73bea60d9d460af773f28ddf81058870a278eb37addce371
|
7
|
+
data.tar.gz: f8dddc93793e276ba0efe55875b1054ab5ba37c46f4e0bceba09dee39a91fcc05ddb17642db1a8ff854317cb8536374b72a0965a95b4bcbf498a7209a6587031
|
data/ext/hermann/hermann_lib.c
CHANGED
@@ -379,13 +379,12 @@ void consumer_consume_loop(HermannInstanceConfig* consumerConfig) {
|
|
379
379
|
}
|
380
380
|
|
381
381
|
/**
|
382
|
-
* Hermann::Consumer.consume
|
383
|
-
*
|
384
|
-
* Begin listening on the configured topic for messages. msg_consume will be called on each message received.
|
382
|
+
* Hermann::Lib::Consumer.consume
|
385
383
|
*
|
386
384
|
* @param VALUE self the Ruby object for this consumer
|
385
|
+
* @param VALUE topic the Ruby string representing a topic to consume
|
387
386
|
*/
|
388
|
-
static VALUE consumer_consume(VALUE self) {
|
387
|
+
static VALUE consumer_consume(VALUE self, VALUE topic) {
|
389
388
|
|
390
389
|
HermannInstanceConfig* consumerConfig;
|
391
390
|
|
@@ -1012,7 +1011,7 @@ void Init_hermann_lib() {
|
|
1012
1011
|
rb_define_method(c_consumer, "initialize_copy", consumer_init_copy, 1);
|
1013
1012
|
|
1014
1013
|
/* Consumer has method 'consume' */
|
1015
|
-
rb_define_method( c_consumer, "consume", consumer_consume,
|
1014
|
+
rb_define_method( c_consumer, "consume", consumer_consume, 1);
|
1016
1015
|
|
1017
1016
|
/* ---- Define the producer class ---- */
|
1018
1017
|
c_producer = rb_define_class_under(lib_module, "Producer", rb_cObject);
|
data/lib/hermann.rb
CHANGED
@@ -2,19 +2,34 @@ module Hermann
|
|
2
2
|
def self.jruby?
|
3
3
|
return RUBY_PLATFORM == "java"
|
4
4
|
end
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
5
|
+
# Validates that the args are non-blank strings
|
6
|
+
#
|
7
|
+
# @param [Object] key to validate
|
8
|
+
#
|
9
|
+
# @param [Object] val to validate
|
10
|
+
#
|
11
|
+
# @raises [ConfigurationErorr] if either values are empty
|
12
|
+
def self.validate_property!(key, val)
|
13
|
+
if key.to_s.empty? || val.to_s.empty?
|
14
|
+
raise Hermann::Errors::ConfigurationError
|
12
15
|
end
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
16
|
+
end
|
17
|
+
|
18
|
+
# Packages options into Java Properties object
|
19
|
+
#
|
20
|
+
# @params [Hash] hash of options to package
|
21
|
+
#
|
22
|
+
# @return [Properties] packaged java properties
|
23
|
+
def self.package_properties(options)
|
24
|
+
properties = JavaUtil::Properties.new
|
25
|
+
options.each do |key, val|
|
26
|
+
Hermann.validate_property!(key, val)
|
27
|
+
properties.put(key, val)
|
18
28
|
end
|
29
|
+
properties
|
19
30
|
end
|
20
31
|
end
|
32
|
+
|
33
|
+
if Hermann.jruby?
|
34
|
+
require 'hermann/java'
|
35
|
+
end
|
data/lib/hermann/consumer.rb
CHANGED
@@ -1,24 +1,57 @@
|
|
1
1
|
require 'hermann'
|
2
2
|
|
3
|
-
|
3
|
+
if Hermann.jruby?
|
4
|
+
require 'hermann/provider/java_simple_consumer'
|
5
|
+
else
|
4
6
|
require 'hermann_lib'
|
5
7
|
end
|
6
8
|
|
7
9
|
module Hermann
|
10
|
+
# Hermann::Consumer provides a simple consumer API which is only safe to be
|
11
|
+
# executed in a single thread
|
8
12
|
class Consumer
|
9
13
|
attr_reader :topic, :brokers, :partition, :internal
|
10
14
|
|
11
|
-
|
15
|
+
|
16
|
+
# Instantiate Consumer
|
17
|
+
#
|
18
|
+
# @params [String] kafka topic
|
19
|
+
#
|
20
|
+
# @params [String] group ID
|
21
|
+
#
|
22
|
+
# @params [String] comma separated zookeeper list
|
23
|
+
#
|
24
|
+
# @params [Hash] options for consumer
|
25
|
+
# @option opts [String] :brokers (for MRI) Comma separated list of brokers
|
26
|
+
# @option opts [String] :partition (for MRI) The kafka partition
|
27
|
+
# @option opts [Fixnum] :sleep_time (Jruby) Time to sleep between consume retries, defaults to 1sec
|
28
|
+
# @option opts [Boolean] :do_retry (Jruby) Retry consume attempts if exceptions are thrown, defaults to true
|
29
|
+
def initialize(topic, groupId, zookeepers, opts={})
|
12
30
|
@topic = topic
|
13
31
|
@brokers = brokers
|
14
32
|
@partition = partition
|
15
|
-
|
33
|
+
|
34
|
+
if Hermann.jruby?
|
35
|
+
@internal = Hermann::Provider::JavaSimpleConsumer.new(zookeepers, groupId, topic, opts)
|
36
|
+
else
|
37
|
+
brokers = opts.delete(:brokers)
|
38
|
+
partition = opts.delete(:partition)
|
16
39
|
@internal = Hermann::Lib::Consumer.new(topic, brokers, partition)
|
17
40
|
end
|
18
41
|
end
|
19
42
|
|
20
|
-
|
21
|
-
|
43
|
+
# Delegates the consume method to internal consumer classes
|
44
|
+
def consume(topic=nil, &block)
|
45
|
+
@internal.consume(topic, &block)
|
46
|
+
end
|
47
|
+
|
48
|
+
# Delegates the shutdown of kafka messages threads to internal consumer classes
|
49
|
+
def shutdown
|
50
|
+
if Hermann.jruby?
|
51
|
+
@internal.shutdown
|
52
|
+
else
|
53
|
+
#no op
|
54
|
+
end
|
22
55
|
end
|
23
56
|
end
|
24
57
|
end
|
data/lib/hermann/errors.rb
CHANGED
@@ -2,7 +2,20 @@
|
|
2
2
|
module Hermann
|
3
3
|
module Errors
|
4
4
|
# Error for connectivity problems with the Kafka brokers
|
5
|
-
class ConnectivityError
|
5
|
+
class ConnectivityError < StandardError
|
6
|
+
attr_reader :java_exception
|
7
|
+
|
8
|
+
# Initialize a connectivity error
|
9
|
+
#
|
10
|
+
# @param [String] message Exception's message
|
11
|
+
# @param [Hash[ options
|
12
|
+
# @option options [Java::Lang::RuntimeException] :java_exception An
|
13
|
+
# underlying Java exception
|
14
|
+
def initialize(message, options={})
|
15
|
+
super(message)
|
16
|
+
@java_exception = options[:java_exception]
|
17
|
+
end
|
18
|
+
end
|
6
19
|
|
7
20
|
# For passing incorrect config and options to kafka
|
8
21
|
class ConfigurationError < StandardError; end
|
data/lib/hermann/java.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
module Hermann
|
2
|
+
require 'java'
|
3
|
+
require 'hermann_jars'
|
4
|
+
|
5
|
+
module JavaUtil
|
6
|
+
include_package 'java.util'
|
7
|
+
end
|
8
|
+
module ProducerUtil
|
9
|
+
include_package 'kafka.producer'
|
10
|
+
end
|
11
|
+
module ConsumerUtil
|
12
|
+
include_package "kafka.consumer"
|
13
|
+
end
|
14
|
+
module JavaApiUtil
|
15
|
+
include_package 'kafka.javaapi.producer'
|
16
|
+
end
|
17
|
+
module KafkaUtil
|
18
|
+
include_package "kafka.util"
|
19
|
+
end
|
20
|
+
end
|
data/lib/hermann/producer.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
|
+
require 'thread_safe'
|
2
|
+
|
1
3
|
require 'hermann'
|
2
4
|
require 'hermann/result'
|
3
5
|
|
4
|
-
|
5
6
|
if RUBY_PLATFORM == "java"
|
6
7
|
require 'hermann/provider/java_producer'
|
7
8
|
else
|
@@ -18,15 +19,15 @@ module Hermann
|
|
18
19
|
# @param [Array] brokers An array of "host:port" strings for the brokers
|
19
20
|
def initialize(topic, brokers, opts={})
|
20
21
|
@topic = topic
|
21
|
-
@brokers = brokers
|
22
|
-
if
|
22
|
+
@brokers = ThreadSafe::Array.new(brokers)
|
23
|
+
if Hermann.jruby?
|
23
24
|
@internal = Hermann::Provider::JavaProducer.new(brokers, opts)
|
24
25
|
else
|
25
|
-
@internal = Hermann::Lib::Producer.new(brokers)
|
26
|
+
@internal = Hermann::Lib::Producer.new(brokers.join(','))
|
26
27
|
end
|
27
28
|
# We're tracking children so we can make sure that at Producer exit we
|
28
29
|
# make a reasonable attempt to clean up outstanding result objects
|
29
|
-
@children =
|
30
|
+
@children = ThreadSafe::Array.new
|
30
31
|
end
|
31
32
|
|
32
33
|
# @return [Boolean] True if our underlying producer object thinks it's
|
@@ -60,7 +61,7 @@ module Hermann
|
|
60
61
|
return value.map { |e| self.push(e, opts) }
|
61
62
|
end
|
62
63
|
|
63
|
-
if
|
64
|
+
if Hermann.jruby?
|
64
65
|
result = @internal.push_single(value, topic, nil)
|
65
66
|
unless result.nil?
|
66
67
|
@children << result
|
@@ -69,6 +70,9 @@ module Hermann
|
|
69
70
|
# called correctly and we don't leak memory
|
70
71
|
reap_children
|
71
72
|
else
|
73
|
+
# Ticking reactor to make sure that we don't inadvertantly let the
|
74
|
+
# librdkafka callback queue overflow
|
75
|
+
tick_reactor
|
72
76
|
result = create_result
|
73
77
|
@internal.push_single(value, topic, result)
|
74
78
|
end
|
@@ -118,7 +122,6 @@ module Hermann
|
|
118
122
|
return (total_children - children.size)
|
119
123
|
end
|
120
124
|
|
121
|
-
|
122
125
|
private
|
123
126
|
|
124
127
|
def rounded_timeout(timeout)
|
@@ -11,6 +11,13 @@ module Hermann
|
|
11
11
|
class JavaProducer
|
12
12
|
attr_accessor :producer
|
13
13
|
|
14
|
+
#default kafka Producer options
|
15
|
+
DEFAULTS = {
|
16
|
+
'serializer.class' => 'kafka.serializer.StringEncoder',
|
17
|
+
'partitioner.class' => 'kafka.producer.DefaultPartitioner',
|
18
|
+
'request.required.acks' => '1',
|
19
|
+
'message.send.max.retries' => '0'
|
20
|
+
}.freeze
|
14
21
|
|
15
22
|
# Instantiate JavaProducer
|
16
23
|
#
|
@@ -25,17 +32,10 @@ module Hermann
|
|
25
32
|
# JavaProducer.new('0:9092', {'request.required.acks' => '1'})
|
26
33
|
#
|
27
34
|
def initialize(brokers, opts={})
|
28
|
-
|
29
|
-
config = create_config(properties)
|
35
|
+
config = create_config(brokers, opts)
|
30
36
|
@producer = JavaApiUtil::Producer.new(config)
|
31
37
|
end
|
32
38
|
|
33
|
-
DEFAULTS = {
|
34
|
-
'serializer.class' => 'kafka.serializer.StringEncoder',
|
35
|
-
'partitioner.class' => 'kafka.producer.DefaultPartitioner',
|
36
|
-
'request.required.acks' => '1'
|
37
|
-
}.freeze
|
38
|
-
|
39
39
|
# Push a value onto the Kafka topic passed to this +Producer+
|
40
40
|
#
|
41
41
|
# @param [Object] value A single object to push
|
@@ -47,7 +47,12 @@ module Hermann
|
|
47
47
|
def push_single(msg, topic, unused)
|
48
48
|
Concurrent::Promise.execute {
|
49
49
|
data = ProducerUtil::KeyedMessage.new(topic, msg)
|
50
|
-
|
50
|
+
begin
|
51
|
+
@producer.send(data)
|
52
|
+
rescue Java::KafkaCommon::FailedToSendMessageException => jexc
|
53
|
+
raise Hermann::Errors::ConnectivityError.new(jexc.message,
|
54
|
+
:java_exception => jexc)
|
55
|
+
end
|
51
56
|
}
|
52
57
|
end
|
53
58
|
|
@@ -67,39 +72,21 @@ module Hermann
|
|
67
72
|
end
|
68
73
|
|
69
74
|
private
|
70
|
-
|
71
75
|
# Creates a ProducerConfig object
|
72
76
|
#
|
73
|
-
# @param [
|
74
|
-
#
|
75
|
-
# @return [ProducerConfig] - packaged config for +Producer+
|
76
|
-
def create_config(properties)
|
77
|
-
ProducerUtil::ProducerConfig.new(properties)
|
78
|
-
end
|
79
|
-
|
80
|
-
# Creates Properties Object
|
77
|
+
# @param [String] comma separated list of brokers
|
81
78
|
#
|
82
79
|
# @param [Hash] brokers passed into this function
|
83
80
|
# @option args [String] :brokers - string of brokers
|
84
81
|
#
|
85
|
-
# @return [
|
82
|
+
# @return [ProducerConfig] - packaged config for +Producer+
|
86
83
|
#
|
87
84
|
# @raises [RuntimeError] if options does not contain key value strings
|
88
|
-
def
|
89
|
-
brokers
|
90
|
-
options
|
91
|
-
properties =
|
92
|
-
|
93
|
-
validate_property!(key, val)
|
94
|
-
properties.put(key, val)
|
95
|
-
end
|
96
|
-
properties
|
97
|
-
end
|
98
|
-
|
99
|
-
def validate_property!(key, val)
|
100
|
-
if key.to_s.empty? || val.to_s.empty?
|
101
|
-
raise Hermann::Errors::ConfigurationError, "Invalid Broker Properties"
|
102
|
-
end
|
85
|
+
def create_config(brokers, opts={})
|
86
|
+
brokers = { 'metadata.broker.list' => brokers }
|
87
|
+
options = DEFAULTS.merge(brokers).merge(opts)
|
88
|
+
properties = Hermann.package_properties(options)
|
89
|
+
ProducerUtil::ProducerConfig.new(properties)
|
103
90
|
end
|
104
91
|
end
|
105
92
|
end
|
@@ -0,0 +1,122 @@
|
|
1
|
+
require 'hermann'
|
2
|
+
require 'hermann/errors'
|
3
|
+
|
4
|
+
module Hermann
|
5
|
+
module Provider
|
6
|
+
|
7
|
+
# Implements a java based consumer. The #consumer method loops infinitely,
|
8
|
+
# the hasNext() method blocks until a message is available.
|
9
|
+
class JavaSimpleConsumer
|
10
|
+
attr_accessor :consumer, :topic, :zookeeper
|
11
|
+
|
12
|
+
NUM_THREADS = 1
|
13
|
+
|
14
|
+
#default zookeeper connection options
|
15
|
+
DEFAULTS = {
|
16
|
+
'zookeeper.session.timeout.ms' => '400',
|
17
|
+
'zookeeper.sync.time.ms' => '200',
|
18
|
+
'auto.commit.interval.ms' => '1000'
|
19
|
+
}.freeze
|
20
|
+
|
21
|
+
# Instantiate JavaSimpleConsumer
|
22
|
+
#
|
23
|
+
# @params [String] list of zookeepers
|
24
|
+
#
|
25
|
+
# @params [String] Group ID
|
26
|
+
#
|
27
|
+
# @params [String] Kafka topic
|
28
|
+
#
|
29
|
+
# @params [Hash] kafka options for consumer
|
30
|
+
# @option opts [Fixnum] :sleep_time Time to sleep between consume retries, defaults to 1sec
|
31
|
+
# @option opts [Boolean] :do_retry Retry consume attempts if exceptions are thrown, defaults to true
|
32
|
+
def initialize(zookeepers, groupId, topic, opts={})
|
33
|
+
config = create_config(zookeepers, groupId)
|
34
|
+
@consumer = ConsumerUtil::Consumer.createJavaConsumerConnector(config)
|
35
|
+
@topic = topic
|
36
|
+
@sleep_time = opts.delete(:sleep_time) || 1
|
37
|
+
@do_retry = opts.delete(:do_retry) || true
|
38
|
+
end
|
39
|
+
|
40
|
+
# Shuts down the various threads created by createMessageStreams
|
41
|
+
# This can be called after the thread executing consume has exited
|
42
|
+
# to clean up.
|
43
|
+
def shutdown
|
44
|
+
@consumer.shutdown
|
45
|
+
end
|
46
|
+
|
47
|
+
# Starts infinite loop to consume messages. hasNext() blocks until a
|
48
|
+
# message is available at which point it is yielded to the block
|
49
|
+
#
|
50
|
+
# @params [String] optional topic to override initialized topic
|
51
|
+
#
|
52
|
+
# ==== Examples
|
53
|
+
#
|
54
|
+
# consumer.consume do |message|
|
55
|
+
# puts "Received: #{message}"
|
56
|
+
# end
|
57
|
+
#
|
58
|
+
def consume(topic=nil)
|
59
|
+
begin
|
60
|
+
stream = get_stream(topic)
|
61
|
+
it = stream.iterator
|
62
|
+
while it.hasNext do
|
63
|
+
yield it.next.message.to_s
|
64
|
+
end
|
65
|
+
rescue Exception => e
|
66
|
+
puts "#{self.class.name}#consume exception: #{e.class.name}"
|
67
|
+
puts "Msg: #{e.message}"
|
68
|
+
puts e.backtrace.join("\n")
|
69
|
+
if retry?
|
70
|
+
sleep @sleep_time
|
71
|
+
retry
|
72
|
+
else
|
73
|
+
raise e
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
private
|
79
|
+
def retry?
|
80
|
+
@do_retry
|
81
|
+
end
|
82
|
+
|
83
|
+
# Gets the message stream of the topic. Creates message streams for
|
84
|
+
# a topic and the number of threads requested. In this case the default
|
85
|
+
# number of threads is NUM_THREADS.
|
86
|
+
#
|
87
|
+
# @params [String] optional topic to override initialized topic
|
88
|
+
def get_stream(topic)
|
89
|
+
current_topic = topic || @topic
|
90
|
+
@topicCountMap = JavaUtil::HashMap.new
|
91
|
+
@value = NUM_THREADS.to_java Java::int
|
92
|
+
@topicCountMap.put("#{current_topic}", @value)
|
93
|
+
consumerMap = @consumer.createMessageStreams(@topicCountMap)
|
94
|
+
consumerMap[current_topic].first
|
95
|
+
end
|
96
|
+
|
97
|
+
# Creates a ConsumerConfig object
|
98
|
+
#
|
99
|
+
# @param [String] zookeepers list
|
100
|
+
#
|
101
|
+
# @param [String] group ID
|
102
|
+
#
|
103
|
+
# @return [ConsumerConfig] - packaged config for +Consumer+
|
104
|
+
#
|
105
|
+
# @raises [RuntimeError] if options does not contain key value strings
|
106
|
+
def create_config(zookeepers, groupId, opts={})
|
107
|
+
config = connect_opts(zookeepers, groupId)
|
108
|
+
options = DEFAULTS.merge(config).merge(opts)
|
109
|
+
properties = Hermann.package_properties(options)
|
110
|
+
ConsumerUtil::ConsumerConfig.new(properties)
|
111
|
+
end
|
112
|
+
|
113
|
+
# Connection options to pass to ConsumerConfig
|
114
|
+
def connect_opts(zookeepers, groupId)
|
115
|
+
{
|
116
|
+
'zookeeper.connect' => zookeepers,
|
117
|
+
'group.id' => groupId
|
118
|
+
}
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
data/lib/hermann/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: hermann
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.19.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- R. Tyler Croy
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2014-10-
|
12
|
+
date: 2014-10-29 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -39,6 +39,20 @@ dependencies:
|
|
39
39
|
- - ~>
|
40
40
|
- !ruby/object:Gem::Version
|
41
41
|
version: 1.9.4
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ~>
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.3.4
|
48
|
+
name: thread_safe
|
49
|
+
prerelease: false
|
50
|
+
type: :runtime
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - ~>
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: 0.3.4
|
42
56
|
- !ruby/object:Gem::Dependency
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|
44
58
|
requirements:
|
@@ -98,8 +112,10 @@ files:
|
|
98
112
|
- lib/hermann/consumer.rb
|
99
113
|
- lib/hermann/discovery/zookeeper.rb
|
100
114
|
- lib/hermann/errors.rb
|
115
|
+
- lib/hermann/java.rb
|
101
116
|
- lib/hermann/producer.rb
|
102
117
|
- lib/hermann/provider/java_producer.rb
|
118
|
+
- lib/hermann/provider/java_simple_consumer.rb
|
103
119
|
- lib/hermann/result.rb
|
104
120
|
- lib/hermann/timeout.rb
|
105
121
|
- lib/hermann/version.rb
|