hermann 0.18.1-java → 0.19.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 6da9dc68c659381482844060d7534fb070fde17e
4
- data.tar.gz: 74ea4247774c4cabe6880b1db45a0234b9a8ab83
3
+ metadata.gz: a6ae048936233d039e0422d5fcf03deff436b95a
4
+ data.tar.gz: c9d8807df3dd9d8e9efd089a325471b24a27bb3f
5
5
  SHA512:
6
- metadata.gz: c32c2668dcb9ccfc6f6768e3cdef001c450f29ae45887b262ca5b43bccd1f3817658bee048889b1a5af5ace3ce6761c06da79903f93394d8fa3e30beeb160da8
7
- data.tar.gz: a164452507a36f750f8e480806fc5713b715b070e26c73bfa25c3931f7944f09c736fcc48e271615f138a248d9469b3b131993340dafb54fc8e2793b03512988
6
+ metadata.gz: c38aeb5099c6cf1f33b262b3a9f2b552ab964fc35d3c19a48280005580cf8d6b74401e15352ad93f73bea60d9d460af773f28ddf81058870a278eb37addce371
7
+ data.tar.gz: f8dddc93793e276ba0efe55875b1054ab5ba37c46f4e0bceba09dee39a91fcc05ddb17642db1a8ff854317cb8536374b72a0965a95b4bcbf498a7209a6587031
@@ -379,13 +379,12 @@ void consumer_consume_loop(HermannInstanceConfig* consumerConfig) {
379
379
  }
380
380
 
381
381
  /**
382
- * Hermann::Consumer.consume
383
- *
384
- * Begin listening on the configured topic for messages. msg_consume will be called on each message received.
382
+ * Hermann::Lib::Consumer.consume
385
383
  *
386
384
  * @param VALUE self the Ruby object for this consumer
385
+ * @param VALUE topic the Ruby string representing a topic to consume
387
386
  */
388
- static VALUE consumer_consume(VALUE self) {
387
+ static VALUE consumer_consume(VALUE self, VALUE topic) {
389
388
 
390
389
  HermannInstanceConfig* consumerConfig;
391
390
 
@@ -1012,7 +1011,7 @@ void Init_hermann_lib() {
1012
1011
  rb_define_method(c_consumer, "initialize_copy", consumer_init_copy, 1);
1013
1012
 
1014
1013
  /* Consumer has method 'consume' */
1015
- rb_define_method( c_consumer, "consume", consumer_consume, 0 );
1014
+ rb_define_method( c_consumer, "consume", consumer_consume, 1);
1016
1015
 
1017
1016
  /* ---- Define the producer class ---- */
1018
1017
  c_producer = rb_define_class_under(lib_module, "Producer", rb_cObject);
data/lib/hermann.rb CHANGED
@@ -2,19 +2,34 @@ module Hermann
2
2
  def self.jruby?
3
3
  return RUBY_PLATFORM == "java"
4
4
  end
5
-
6
- if self.jruby?
7
- require 'java'
8
- require 'hermann_jars'
9
-
10
- module JavaUtil
11
- include_package 'java.util'
5
+ # Validates that the args are non-blank strings
6
+ #
7
+ # @param [Object] key to validate
8
+ #
9
+ # @param [Object] val to validate
10
+ #
11
+ # @raises [ConfigurationErorr] if either values are empty
12
+ def self.validate_property!(key, val)
13
+ if key.to_s.empty? || val.to_s.empty?
14
+ raise Hermann::Errors::ConfigurationError
12
15
  end
13
- module ProducerUtil
14
- include_package 'kafka.producer'
15
- end
16
- module JavaApiUtil
17
- include_package 'kafka.javaapi.producer'
16
+ end
17
+
18
+ # Packages options into Java Properties object
19
+ #
20
+ # @params [Hash] hash of options to package
21
+ #
22
+ # @return [Properties] packaged java properties
23
+ def self.package_properties(options)
24
+ properties = JavaUtil::Properties.new
25
+ options.each do |key, val|
26
+ Hermann.validate_property!(key, val)
27
+ properties.put(key, val)
18
28
  end
29
+ properties
19
30
  end
20
31
  end
32
+
33
+ if Hermann.jruby?
34
+ require 'hermann/java'
35
+ end
@@ -1,24 +1,57 @@
1
1
  require 'hermann'
2
2
 
3
- unless Hermann.jruby?
3
+ if Hermann.jruby?
4
+ require 'hermann/provider/java_simple_consumer'
5
+ else
4
6
  require 'hermann_lib'
5
7
  end
6
8
 
7
9
  module Hermann
10
+ # Hermann::Consumer provides a simple consumer API which is only safe to be
11
+ # executed in a single thread
8
12
  class Consumer
9
13
  attr_reader :topic, :brokers, :partition, :internal
10
14
 
11
- def initialize(topic, brokers, partition)
15
+
16
+ # Instantiate Consumer
17
+ #
18
+ # @params [String] kafka topic
19
+ #
20
+ # @params [String] group ID
21
+ #
22
+ # @params [String] comma separated zookeeper list
23
+ #
24
+ # @params [Hash] options for consumer
25
+ # @option opts [String] :brokers (for MRI) Comma separated list of brokers
26
+ # @option opts [String] :partition (for MRI) The kafka partition
27
+ # @option opts [Fixnum] :sleep_time (Jruby) Time to sleep between consume retries, defaults to 1sec
28
+ # @option opts [Boolean] :do_retry (Jruby) Retry consume attempts if exceptions are thrown, defaults to true
29
+ def initialize(topic, groupId, zookeepers, opts={})
12
30
  @topic = topic
13
31
  @brokers = brokers
14
32
  @partition = partition
15
- unless Hermann.jruby?
33
+
34
+ if Hermann.jruby?
35
+ @internal = Hermann::Provider::JavaSimpleConsumer.new(zookeepers, groupId, topic, opts)
36
+ else
37
+ brokers = opts.delete(:brokers)
38
+ partition = opts.delete(:partition)
16
39
  @internal = Hermann::Lib::Consumer.new(topic, brokers, partition)
17
40
  end
18
41
  end
19
42
 
20
- def consume(&block)
21
- @internal.consume(&block)
43
+ # Delegates the consume method to internal consumer classes
44
+ def consume(topic=nil, &block)
45
+ @internal.consume(topic, &block)
46
+ end
47
+
48
+ # Delegates the shutdown of kafka messages threads to internal consumer classes
49
+ def shutdown
50
+ if Hermann.jruby?
51
+ @internal.shutdown
52
+ else
53
+ #no op
54
+ end
22
55
  end
23
56
  end
24
57
  end
@@ -2,7 +2,20 @@
2
2
  module Hermann
3
3
  module Errors
4
4
  # Error for connectivity problems with the Kafka brokers
5
- class ConnectivityError; end;
5
+ class ConnectivityError < StandardError
6
+ attr_reader :java_exception
7
+
8
+ # Initialize a connectivity error
9
+ #
10
+ # @param [String] message Exception's message
11
+ # @param [Hash[ options
12
+ # @option options [Java::Lang::RuntimeException] :java_exception An
13
+ # underlying Java exception
14
+ def initialize(message, options={})
15
+ super(message)
16
+ @java_exception = options[:java_exception]
17
+ end
18
+ end
6
19
 
7
20
  # For passing incorrect config and options to kafka
8
21
  class ConfigurationError < StandardError; end
@@ -0,0 +1,20 @@
1
+ module Hermann
2
+ require 'java'
3
+ require 'hermann_jars'
4
+
5
+ module JavaUtil
6
+ include_package 'java.util'
7
+ end
8
+ module ProducerUtil
9
+ include_package 'kafka.producer'
10
+ end
11
+ module ConsumerUtil
12
+ include_package "kafka.consumer"
13
+ end
14
+ module JavaApiUtil
15
+ include_package 'kafka.javaapi.producer'
16
+ end
17
+ module KafkaUtil
18
+ include_package "kafka.util"
19
+ end
20
+ end
@@ -1,7 +1,8 @@
1
+ require 'thread_safe'
2
+
1
3
  require 'hermann'
2
4
  require 'hermann/result'
3
5
 
4
-
5
6
  if RUBY_PLATFORM == "java"
6
7
  require 'hermann/provider/java_producer'
7
8
  else
@@ -18,15 +19,15 @@ module Hermann
18
19
  # @param [Array] brokers An array of "host:port" strings for the brokers
19
20
  def initialize(topic, brokers, opts={})
20
21
  @topic = topic
21
- @brokers = brokers
22
- if RUBY_PLATFORM == "java"
22
+ @brokers = ThreadSafe::Array.new(brokers)
23
+ if Hermann.jruby?
23
24
  @internal = Hermann::Provider::JavaProducer.new(brokers, opts)
24
25
  else
25
- @internal = Hermann::Lib::Producer.new(brokers)
26
+ @internal = Hermann::Lib::Producer.new(brokers.join(','))
26
27
  end
27
28
  # We're tracking children so we can make sure that at Producer exit we
28
29
  # make a reasonable attempt to clean up outstanding result objects
29
- @children = []
30
+ @children = ThreadSafe::Array.new
30
31
  end
31
32
 
32
33
  # @return [Boolean] True if our underlying producer object thinks it's
@@ -60,7 +61,7 @@ module Hermann
60
61
  return value.map { |e| self.push(e, opts) }
61
62
  end
62
63
 
63
- if RUBY_PLATFORM == "java"
64
+ if Hermann.jruby?
64
65
  result = @internal.push_single(value, topic, nil)
65
66
  unless result.nil?
66
67
  @children << result
@@ -69,6 +70,9 @@ module Hermann
69
70
  # called correctly and we don't leak memory
70
71
  reap_children
71
72
  else
73
+ # Ticking reactor to make sure that we don't inadvertantly let the
74
+ # librdkafka callback queue overflow
75
+ tick_reactor
72
76
  result = create_result
73
77
  @internal.push_single(value, topic, result)
74
78
  end
@@ -118,7 +122,6 @@ module Hermann
118
122
  return (total_children - children.size)
119
123
  end
120
124
 
121
-
122
125
  private
123
126
 
124
127
  def rounded_timeout(timeout)
@@ -11,6 +11,13 @@ module Hermann
11
11
  class JavaProducer
12
12
  attr_accessor :producer
13
13
 
14
+ #default kafka Producer options
15
+ DEFAULTS = {
16
+ 'serializer.class' => 'kafka.serializer.StringEncoder',
17
+ 'partitioner.class' => 'kafka.producer.DefaultPartitioner',
18
+ 'request.required.acks' => '1',
19
+ 'message.send.max.retries' => '0'
20
+ }.freeze
14
21
 
15
22
  # Instantiate JavaProducer
16
23
  #
@@ -25,17 +32,10 @@ module Hermann
25
32
  # JavaProducer.new('0:9092', {'request.required.acks' => '1'})
26
33
  #
27
34
  def initialize(brokers, opts={})
28
- properties = create_properties(brokers, opts)
29
- config = create_config(properties)
35
+ config = create_config(brokers, opts)
30
36
  @producer = JavaApiUtil::Producer.new(config)
31
37
  end
32
38
 
33
- DEFAULTS = {
34
- 'serializer.class' => 'kafka.serializer.StringEncoder',
35
- 'partitioner.class' => 'kafka.producer.DefaultPartitioner',
36
- 'request.required.acks' => '1'
37
- }.freeze
38
-
39
39
  # Push a value onto the Kafka topic passed to this +Producer+
40
40
  #
41
41
  # @param [Object] value A single object to push
@@ -47,7 +47,12 @@ module Hermann
47
47
  def push_single(msg, topic, unused)
48
48
  Concurrent::Promise.execute {
49
49
  data = ProducerUtil::KeyedMessage.new(topic, msg)
50
- @producer.send(data)
50
+ begin
51
+ @producer.send(data)
52
+ rescue Java::KafkaCommon::FailedToSendMessageException => jexc
53
+ raise Hermann::Errors::ConnectivityError.new(jexc.message,
54
+ :java_exception => jexc)
55
+ end
51
56
  }
52
57
  end
53
58
 
@@ -67,39 +72,21 @@ module Hermann
67
72
  end
68
73
 
69
74
  private
70
-
71
75
  # Creates a ProducerConfig object
72
76
  #
73
- # @param [Properties] object with broker properties
74
- #
75
- # @return [ProducerConfig] - packaged config for +Producer+
76
- def create_config(properties)
77
- ProducerUtil::ProducerConfig.new(properties)
78
- end
79
-
80
- # Creates Properties Object
77
+ # @param [String] comma separated list of brokers
81
78
  #
82
79
  # @param [Hash] brokers passed into this function
83
80
  # @option args [String] :brokers - string of brokers
84
81
  #
85
- # @return [Properties] properties object for creating +ProducerConfig+
82
+ # @return [ProducerConfig] - packaged config for +Producer+
86
83
  #
87
84
  # @raises [RuntimeError] if options does not contain key value strings
88
- def create_properties(brokers, opts={})
89
- brokers = { 'metadata.broker.list' => brokers }
90
- options = DEFAULTS.merge(brokers).merge(opts)
91
- properties = JavaUtil::Properties.new
92
- options.each do |key, val|
93
- validate_property!(key, val)
94
- properties.put(key, val)
95
- end
96
- properties
97
- end
98
-
99
- def validate_property!(key, val)
100
- if key.to_s.empty? || val.to_s.empty?
101
- raise Hermann::Errors::ConfigurationError, "Invalid Broker Properties"
102
- end
85
+ def create_config(brokers, opts={})
86
+ brokers = { 'metadata.broker.list' => brokers }
87
+ options = DEFAULTS.merge(brokers).merge(opts)
88
+ properties = Hermann.package_properties(options)
89
+ ProducerUtil::ProducerConfig.new(properties)
103
90
  end
104
91
  end
105
92
  end
@@ -0,0 +1,122 @@
1
+ require 'hermann'
2
+ require 'hermann/errors'
3
+
4
+ module Hermann
5
+ module Provider
6
+
7
+ # Implements a java based consumer. The #consumer method loops infinitely,
8
+ # the hasNext() method blocks until a message is available.
9
+ class JavaSimpleConsumer
10
+ attr_accessor :consumer, :topic, :zookeeper
11
+
12
+ NUM_THREADS = 1
13
+
14
+ #default zookeeper connection options
15
+ DEFAULTS = {
16
+ 'zookeeper.session.timeout.ms' => '400',
17
+ 'zookeeper.sync.time.ms' => '200',
18
+ 'auto.commit.interval.ms' => '1000'
19
+ }.freeze
20
+
21
+ # Instantiate JavaSimpleConsumer
22
+ #
23
+ # @params [String] list of zookeepers
24
+ #
25
+ # @params [String] Group ID
26
+ #
27
+ # @params [String] Kafka topic
28
+ #
29
+ # @params [Hash] kafka options for consumer
30
+ # @option opts [Fixnum] :sleep_time Time to sleep between consume retries, defaults to 1sec
31
+ # @option opts [Boolean] :do_retry Retry consume attempts if exceptions are thrown, defaults to true
32
+ def initialize(zookeepers, groupId, topic, opts={})
33
+ config = create_config(zookeepers, groupId)
34
+ @consumer = ConsumerUtil::Consumer.createJavaConsumerConnector(config)
35
+ @topic = topic
36
+ @sleep_time = opts.delete(:sleep_time) || 1
37
+ @do_retry = opts.delete(:do_retry) || true
38
+ end
39
+
40
+ # Shuts down the various threads created by createMessageStreams
41
+ # This can be called after the thread executing consume has exited
42
+ # to clean up.
43
+ def shutdown
44
+ @consumer.shutdown
45
+ end
46
+
47
+ # Starts infinite loop to consume messages. hasNext() blocks until a
48
+ # message is available at which point it is yielded to the block
49
+ #
50
+ # @params [String] optional topic to override initialized topic
51
+ #
52
+ # ==== Examples
53
+ #
54
+ # consumer.consume do |message|
55
+ # puts "Received: #{message}"
56
+ # end
57
+ #
58
+ def consume(topic=nil)
59
+ begin
60
+ stream = get_stream(topic)
61
+ it = stream.iterator
62
+ while it.hasNext do
63
+ yield it.next.message.to_s
64
+ end
65
+ rescue Exception => e
66
+ puts "#{self.class.name}#consume exception: #{e.class.name}"
67
+ puts "Msg: #{e.message}"
68
+ puts e.backtrace.join("\n")
69
+ if retry?
70
+ sleep @sleep_time
71
+ retry
72
+ else
73
+ raise e
74
+ end
75
+ end
76
+ end
77
+
78
+ private
79
+ def retry?
80
+ @do_retry
81
+ end
82
+
83
+ # Gets the message stream of the topic. Creates message streams for
84
+ # a topic and the number of threads requested. In this case the default
85
+ # number of threads is NUM_THREADS.
86
+ #
87
+ # @params [String] optional topic to override initialized topic
88
+ def get_stream(topic)
89
+ current_topic = topic || @topic
90
+ @topicCountMap = JavaUtil::HashMap.new
91
+ @value = NUM_THREADS.to_java Java::int
92
+ @topicCountMap.put("#{current_topic}", @value)
93
+ consumerMap = @consumer.createMessageStreams(@topicCountMap)
94
+ consumerMap[current_topic].first
95
+ end
96
+
97
+ # Creates a ConsumerConfig object
98
+ #
99
+ # @param [String] zookeepers list
100
+ #
101
+ # @param [String] group ID
102
+ #
103
+ # @return [ConsumerConfig] - packaged config for +Consumer+
104
+ #
105
+ # @raises [RuntimeError] if options does not contain key value strings
106
+ def create_config(zookeepers, groupId, opts={})
107
+ config = connect_opts(zookeepers, groupId)
108
+ options = DEFAULTS.merge(config).merge(opts)
109
+ properties = Hermann.package_properties(options)
110
+ ConsumerUtil::ConsumerConfig.new(properties)
111
+ end
112
+
113
+ # Connection options to pass to ConsumerConfig
114
+ def connect_opts(zookeepers, groupId)
115
+ {
116
+ 'zookeeper.connect' => zookeepers,
117
+ 'group.id' => groupId
118
+ }
119
+ end
120
+ end
121
+ end
122
+ end
@@ -1,3 +1,3 @@
1
1
  module Hermann
2
- VERSION = '0.18.1'
2
+ VERSION = '0.19.0'
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: hermann
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.18.1
4
+ version: 0.19.0
5
5
  platform: java
6
6
  authors:
7
7
  - R. Tyler Croy
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2014-10-15 00:00:00.000000000 Z
12
+ date: 2014-10-29 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  requirement: !ruby/object:Gem::Requirement
@@ -39,6 +39,20 @@ dependencies:
39
39
  - - ~>
40
40
  - !ruby/object:Gem::Version
41
41
  version: 1.9.4
42
+ - !ruby/object:Gem::Dependency
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ~>
46
+ - !ruby/object:Gem::Version
47
+ version: 0.3.4
48
+ name: thread_safe
49
+ prerelease: false
50
+ type: :runtime
51
+ version_requirements: !ruby/object:Gem::Requirement
52
+ requirements:
53
+ - - ~>
54
+ - !ruby/object:Gem::Version
55
+ version: 0.3.4
42
56
  - !ruby/object:Gem::Dependency
43
57
  requirement: !ruby/object:Gem::Requirement
44
58
  requirements:
@@ -98,8 +112,10 @@ files:
98
112
  - lib/hermann/consumer.rb
99
113
  - lib/hermann/discovery/zookeeper.rb
100
114
  - lib/hermann/errors.rb
115
+ - lib/hermann/java.rb
101
116
  - lib/hermann/producer.rb
102
117
  - lib/hermann/provider/java_producer.rb
118
+ - lib/hermann/provider/java_simple_consumer.rb
103
119
  - lib/hermann/result.rb
104
120
  - lib/hermann/timeout.rb
105
121
  - lib/hermann/version.rb