hermann 0.22.0-java → 0.23.0.236-java

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: e66cc3de18d3ed620a691c0e308068f471bb89b6
4
- data.tar.gz: 9c1c1171bec630a9cd8ee843e4299bd5d016707a
3
+ metadata.gz: 72692497af5d422995313c36700af4b0cb832411
4
+ data.tar.gz: da5d00bd780a496573cb007a32d6b1e5bea2a775
5
5
  SHA512:
6
- metadata.gz: dcb43908793f2254b2839c59cca84f9e74fc11ae8f06739f984398caf550b223c751891376a9af786963a93c97a1814e9d96db1f7a24095e6b02a87c87c71078
7
- data.tar.gz: 80558b4a81be616687d04ce29e8106c710621db6a4cacb71f1cde3f2a1d9761c785e209fb4dc30738d4da9aaec6d84649d07ba9684fc2bef9745288db03c1d99
6
+ metadata.gz: 2bf5ce855f661db9117bf622de88de8d45233f07c1d3c21282b73052c640cee7d0d86212946301fe21865623955ffa3cb47312927022021e08c66a2b3625acf2
7
+ data.tar.gz: de6b0f7ea494ecc23d1cc95d0e12516aba88f38aa08ed99613ef008dc753bc805147aa7f91768ce90b462eafea9eae4661340f4d9d1185532092e62a4457428d
@@ -146,4 +146,7 @@ dir_config('rdkafka', HEADER_DIRS, LIB_DIRS)
146
146
  # <http://blog.zachallett.com/howto-ruby-c-extension-with-a-static-library>
147
147
  $LOCAL_LIBS << File.join(librdkafka.path, 'lib', 'librdkafka.a')
148
148
 
149
+ have_header('ruby/thread.h')
150
+ have_func('rb_thread_call_without_gvl')
151
+
149
152
  create_makefile('hermann/hermann_lib')
@@ -33,6 +33,11 @@
33
33
 
34
34
  #include "hermann_lib.h"
35
35
 
36
+
37
+ /* how long to let librdkafka block on the socket before returning back to the interpreter.
38
+ * essentially defines how long we wait before consumer_consume_stop_callback() can fire */
39
+ #define CONSUMER_RECVMSG_TIMEOUT_MS 100
40
+
36
41
  /**
37
42
  * Convenience function
38
43
  *
@@ -204,22 +209,16 @@ static void hexdump(FILE *fp,
204
209
  * @param rkmessage rd_kafka_message_t* the message
205
210
  * @param opaque void* opaque context
206
211
  */
207
- static void msg_consume(rd_kafka_message_t *rkmessage,
208
- void *opaque) {
209
-
210
- HermannInstanceConfig* cfg;
211
-
212
- cfg = (HermannInstanceConfig*)opaque;
213
-
212
+ static void msg_consume(rd_kafka_message_t *rkmessage, HermannInstanceConfig *cfg) {
214
213
  if (rkmessage->err) {
215
214
  if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
216
- fprintf(stderr,
217
- "%% Consumer reached end of %s [%"PRId32"] "
218
- "message queue at offset %"PRId64"\n",
219
- rd_kafka_topic_name(rkmessage->rkt),
220
- rkmessage->partition, rkmessage->offset);
221
-
222
215
  if (cfg->exit_eof) {
216
+ fprintf(stderr,
217
+ "%% Consumer reached end of %s [%"PRId32"] "
218
+ "message queue at offset %"PRId64"\n",
219
+ rd_kafka_topic_name(rkmessage->rkt),
220
+ rkmessage->partition, rkmessage->offset);
221
+
223
222
  cfg->run = 0;
224
223
  }
225
224
 
@@ -341,7 +340,7 @@ void consumer_init_kafka(HermannInstanceConfig* config) {
341
340
 
342
341
  // Ruby gem extensions
343
342
 
344
- #ifdef RB_THREAD_BLOCKING_REGION
343
+ #if defined(RB_THREAD_BLOCKING_REGION) || defined(HAVE_RB_THREAD_CALL_WITHOUT_GVL)
345
344
  /* NOTE: We only need this method defined if RB_THREAD_BLOCKING_REGION is
346
345
  * defined, otherwise it's unused
347
346
  */
@@ -360,21 +359,64 @@ static void consumer_consume_stop_callback(void *ptr) {
360
359
  #endif
361
360
 
362
361
  /**
363
- * Loop on a timeout to receive messages from Kafka. When the consumer_consume_stop_callback is invoked by Ruby,
364
- * we'll break out of our loop and return.
362
+ * consumer_recv_msg
363
+ *
364
+ * Consume a single message from the kafka stream. The only function that should be invoked
365
+ * without the GVL held.
366
+ *
367
+ * @param HermannInstanceConfig* The hermann configuration for this consumer
368
+ *
365
369
  */
366
- void consumer_consume_loop(HermannInstanceConfig* consumerConfig) {
367
370
 
371
+ static void *consumer_recv_msg(void *ptr)
372
+ {
373
+ rd_kafka_message_t *ret;
374
+ HermannInstanceConfig *consumerConfig = (HermannInstanceConfig *) ptr;
375
+
376
+ ret = rd_kafka_consume(consumerConfig->rkt, consumerConfig->partition, CONSUMER_RECVMSG_TIMEOUT_MS);
377
+
378
+ if ( ret == NULL ) {
379
+ if ( errno != ETIMEDOUT )
380
+ fprintf(stderr, "%% Error: %s\n", rd_kafka_err2str( rd_kafka_errno2err(errno)));
381
+ }
382
+
383
+ return (void *) ret;
384
+ }
385
+
386
+ /**
387
+ * consumer_consume_loop
388
+ *
389
+ * A timeout-interrupted loop in which we drop the GVL and attemptto receive
390
+ * messages from Kafka. We'll check every CONSUMER_RECVMSG_TIMEOUT_MS, or
391
+ * after every message, to see if the ruby interpreter wants us to exit the
392
+ * loop.
393
+ *
394
+ * @param HermannInstanceConfig* The hermann configuration for this consumer
395
+ */
396
+
397
+ static void consumer_consume_loop(HermannInstanceConfig* consumerConfig) {
398
+ rd_kafka_message_t *msg;
368
399
  TRACER("\n");
369
400
 
370
401
  while (consumerConfig->run) {
371
- if (rd_kafka_consume_callback(consumerConfig->rkt, consumerConfig->partition,
372
- 1000/*timeout*/,
373
- msg_consume,
374
- consumerConfig) < 0) {
375
- fprintf(stderr, "%% Error: %s\n", rd_kafka_err2str( rd_kafka_errno2err(errno)));
376
- }
402
+ #ifdef RB_THREAD_BLOCKING_REGION
403
+ msg = rb_thread_blocking_region(consumer_recv_msg,
404
+ consumerConfig,
405
+ consumer_consume_stop_callback,
406
+ consumerConfig);
407
+ #elif HAVE_RB_THREAD_CALL_WITHOUT_GVL
408
+ msg = rb_thread_call_without_gvl(consumer_recv_msg,
409
+ consumerConfig,
410
+ consumer_consume_stop_callback,
411
+ consumerConfig);
412
+ #else
413
+ msg = consumer_recv_msg(consumerConfig);
414
+ #endif
377
415
 
416
+ if ( msg ) {
417
+ msg_consume(msg, consumerConfig);
418
+ rd_kafka_message_destroy(msg);
419
+ }
378
420
  }
379
421
  }
380
422
 
@@ -412,23 +454,7 @@ static VALUE consumer_consume(VALUE self, VALUE topic) {
412
454
  return Qnil;
413
455
  }
414
456
 
415
- #ifdef RB_THREAD_BLOCKING_REGION
416
- /** The consumer will listen for incoming messages in a loop, timing out and checking the consumerConfig->run
417
- * flag every second.
418
- *
419
- * Call rb_thread_blocking_region to release the GVM lock and allow Ruby to amuse itself while we wait on
420
- * IO from Kafka.
421
- *
422
- * If Ruby needs to interrupt the consumer loop, the stop callback will be invoked and the loop should exit.
423
- */
424
- rb_thread_blocking_region(consumer_consume_loop,
425
- consumerConfig,
426
- consumer_consume_stop_callback,
427
- consumerConfig);
428
- #else
429
- consumer_consume_loop(consumerConfig);
430
- #endif
431
-
457
+ consumer_consume_loop(consumerConfig);
432
458
 
433
459
  /* Stop consuming */
434
460
  rd_kafka_consume_stop(consumerConfig->rkt, consumerConfig->partition);
@@ -669,7 +695,7 @@ static VALUE producer_connect(VALUE self, VALUE timeout) {
669
695
  &data,
670
696
  timeout_ms);
671
697
  TRACER("err: %s (%i)\n", rd_kafka_err2str(err), err);
672
-
698
+
673
699
  if (RD_KAFKA_RESP_ERR_NO_ERROR == err) {
674
700
  TRACER("brokers: %i, topics: %i\n",
675
701
  data->broker_cnt,
@@ -32,6 +32,10 @@
32
32
 
33
33
  #include <ruby.h>
34
34
 
35
+ #ifdef HAVE_RUBY_THREAD_H
36
+ #include <ruby/thread.h>
37
+ #endif
38
+
35
39
  #include <ctype.h>
36
40
  #include <signal.h>
37
41
  #include <string.h>
@@ -10,30 +10,27 @@ module Hermann
10
10
  # Hermann::Consumer provides a simple consumer API which is only safe to be
11
11
  # executed in a single thread
12
12
  class Consumer
13
- attr_reader :topic, :brokers, :partition, :internal
13
+ attr_reader :topic, :internal
14
14
 
15
15
 
16
16
  # Instantiate Consumer
17
17
  #
18
18
  # @params [String] kafka topic
19
- #
20
- # @params [String] group ID
21
- #
22
- # @params [String] comma separated zookeeper list
23
- #
24
19
  # @params [Hash] options for Consumer
25
- # @option opts [String] :brokers (for MRI) Comma separated list of brokers
26
- # @option opts [Integer] :partition (for MRI) The kafka partition
27
- def initialize(topic, groupId, zookeepers, opts={})
20
+ # @option opts [String] :brokers (for MRI) Comma separated list of brokers
21
+ # @option opts [Integer] :partition (for MRI) The kafka partition
22
+ # @option opts [Integer] :zookeepers (for jruby) list of zookeeper servers
23
+ # @option opts [Integer] :group_id (for jruby) client group_id
24
+ #
25
+ def initialize(topic, opts = {})
28
26
  @topic = topic
29
- @brokers = brokers
30
- @partition = partition
31
-
32
27
  if Hermann.jruby?
33
- @internal = Hermann::Provider::JavaSimpleConsumer.new(zookeepers, groupId, topic, opts)
28
+ zookeepers, group_id = require_values_at(opts, :zookeepers, :group_id)
29
+
30
+ @internal = Hermann::Provider::JavaSimpleConsumer.new(zookeepers, group_id, topic, opts)
34
31
  else
35
- brokers = opts.delete(:brokers)
36
- partition = opts.delete(:partition)
32
+ brokers, partition = require_values_at(opts, :brokers, :partition)
33
+
37
34
  @internal = Hermann::Lib::Consumer.new(topic, brokers, partition)
38
35
  end
39
36
  end
@@ -51,5 +48,12 @@ module Hermann
51
48
  #no op
52
49
  end
53
50
  end
51
+
52
+ def require_values_at(opts, *args)
53
+ args.map do |a|
54
+ raise "Please provide :#{a} option!" unless opts[a]
55
+ opts.delete(a)
56
+ end
57
+ end
54
58
  end
55
59
  end
@@ -1,3 +1,3 @@
1
1
  module Hermann
2
- VERSION = '0.22.0'
2
+ VERSION = '0.23.0'
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: hermann
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.22.0
4
+ version: 0.23.0.236
5
5
  platform: java
6
6
  authors:
7
7
  - R. Tyler Croy
@@ -10,50 +10,50 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2015-04-07 00:00:00.000000000 Z
13
+ date: 2015-04-28 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
+ name: concurrent-ruby
17
+ version_requirements: !ruby/object:Gem::Requirement
18
+ requirements:
19
+ - - ~>
20
+ - !ruby/object:Gem::Version
21
+ version: 0.7.0
16
22
  requirement: !ruby/object:Gem::Requirement
17
23
  requirements:
18
24
  - - ~>
19
25
  - !ruby/object:Gem::Version
20
26
  version: 0.7.0
21
- name: concurrent-ruby
22
27
  prerelease: false
23
28
  type: :runtime
29
+ - !ruby/object:Gem::Dependency
30
+ name: thread_safe
24
31
  version_requirements: !ruby/object:Gem::Requirement
25
32
  requirements:
26
33
  - - ~>
27
34
  - !ruby/object:Gem::Version
28
- version: 0.7.0
29
- - !ruby/object:Gem::Dependency
35
+ version: 0.3.4
30
36
  requirement: !ruby/object:Gem::Requirement
31
37
  requirements:
32
38
  - - ~>
33
39
  - !ruby/object:Gem::Version
34
40
  version: 0.3.4
35
- name: thread_safe
36
41
  prerelease: false
37
42
  type: :runtime
43
+ - !ruby/object:Gem::Dependency
44
+ name: jar-dependencies
38
45
  version_requirements: !ruby/object:Gem::Requirement
39
46
  requirements:
40
47
  - - ~>
41
48
  - !ruby/object:Gem::Version
42
- version: 0.3.4
43
- - !ruby/object:Gem::Dependency
49
+ version: 0.1.9
44
50
  requirement: !ruby/object:Gem::Requirement
45
51
  requirements:
46
52
  - - ~>
47
53
  - !ruby/object:Gem::Version
48
54
  version: 0.1.9
49
- name: jar-dependencies
50
55
  prerelease: false
51
56
  type: :runtime
52
- version_requirements: !ruby/object:Gem::Requirement
53
- requirements:
54
- - - ~>
55
- - !ruby/object:Gem::Version
56
- version: 0.1.9
57
57
  description: Ruby gem for talking to Kafka
58
58
  email:
59
59
  - rtyler.croy@lookout.com