hermann 0.22.0-java → 0.23.0.236-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ext/hermann/extconf.rb +3 -0
- data/ext/hermann/hermann_lib.c +67 -41
- data/ext/hermann/hermann_lib.h +4 -0
- data/lib/hermann/consumer.rb +19 -15
- data/lib/hermann/version.rb +1 -1
- metadata +14 -14
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 72692497af5d422995313c36700af4b0cb832411
|
4
|
+
data.tar.gz: da5d00bd780a496573cb007a32d6b1e5bea2a775
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2bf5ce855f661db9117bf622de88de8d45233f07c1d3c21282b73052c640cee7d0d86212946301fe21865623955ffa3cb47312927022021e08c66a2b3625acf2
|
7
|
+
data.tar.gz: de6b0f7ea494ecc23d1cc95d0e12516aba88f38aa08ed99613ef008dc753bc805147aa7f91768ce90b462eafea9eae4661340f4d9d1185532092e62a4457428d
|
data/ext/hermann/extconf.rb
CHANGED
@@ -146,4 +146,7 @@ dir_config('rdkafka', HEADER_DIRS, LIB_DIRS)
|
|
146
146
|
# <http://blog.zachallett.com/howto-ruby-c-extension-with-a-static-library>
|
147
147
|
$LOCAL_LIBS << File.join(librdkafka.path, 'lib', 'librdkafka.a')
|
148
148
|
|
149
|
+
have_header('ruby/thread.h')
|
150
|
+
have_func('rb_thread_call_without_gvl')
|
151
|
+
|
149
152
|
create_makefile('hermann/hermann_lib')
|
data/ext/hermann/hermann_lib.c
CHANGED
@@ -33,6 +33,11 @@
|
|
33
33
|
|
34
34
|
#include "hermann_lib.h"
|
35
35
|
|
36
|
+
|
37
|
+
/* how long to let librdkafka block on the socket before returning back to the interpreter.
|
38
|
+
* essentially defines how long we wait before consumer_consume_stop_callback() can fire */
|
39
|
+
#define CONSUMER_RECVMSG_TIMEOUT_MS 100
|
40
|
+
|
36
41
|
/**
|
37
42
|
* Convenience function
|
38
43
|
*
|
@@ -204,22 +209,16 @@ static void hexdump(FILE *fp,
|
|
204
209
|
* @param rkmessage rd_kafka_message_t* the message
|
205
210
|
* @param opaque void* opaque context
|
206
211
|
*/
|
207
|
-
static void msg_consume(rd_kafka_message_t *rkmessage,
|
208
|
-
void *opaque) {
|
209
|
-
|
210
|
-
HermannInstanceConfig* cfg;
|
211
|
-
|
212
|
-
cfg = (HermannInstanceConfig*)opaque;
|
213
|
-
|
212
|
+
static void msg_consume(rd_kafka_message_t *rkmessage, HermannInstanceConfig *cfg) {
|
214
213
|
if (rkmessage->err) {
|
215
214
|
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
|
216
|
-
fprintf(stderr,
|
217
|
-
"%% Consumer reached end of %s [%"PRId32"] "
|
218
|
-
"message queue at offset %"PRId64"\n",
|
219
|
-
rd_kafka_topic_name(rkmessage->rkt),
|
220
|
-
rkmessage->partition, rkmessage->offset);
|
221
|
-
|
222
215
|
if (cfg->exit_eof) {
|
216
|
+
fprintf(stderr,
|
217
|
+
"%% Consumer reached end of %s [%"PRId32"] "
|
218
|
+
"message queue at offset %"PRId64"\n",
|
219
|
+
rd_kafka_topic_name(rkmessage->rkt),
|
220
|
+
rkmessage->partition, rkmessage->offset);
|
221
|
+
|
223
222
|
cfg->run = 0;
|
224
223
|
}
|
225
224
|
|
@@ -341,7 +340,7 @@ void consumer_init_kafka(HermannInstanceConfig* config) {
|
|
341
340
|
|
342
341
|
// Ruby gem extensions
|
343
342
|
|
344
|
-
#
|
343
|
+
#if defined(RB_THREAD_BLOCKING_REGION) || defined(HAVE_RB_THREAD_CALL_WITHOUT_GVL)
|
345
344
|
/* NOTE: We only need this method defined if RB_THREAD_BLOCKING_REGION is
|
346
345
|
* defined, otherwise it's unused
|
347
346
|
*/
|
@@ -360,21 +359,64 @@ static void consumer_consume_stop_callback(void *ptr) {
|
|
360
359
|
#endif
|
361
360
|
|
362
361
|
/**
|
363
|
-
*
|
364
|
-
*
|
362
|
+
* consumer_recv_msg
|
363
|
+
*
|
364
|
+
* Consume a single message from the kafka stream. The only function that should be invoked
|
365
|
+
* without the GVL held.
|
366
|
+
*
|
367
|
+
* @param HermannInstanceConfig* The hermann configuration for this consumer
|
368
|
+
*
|
365
369
|
*/
|
366
|
-
void consumer_consume_loop(HermannInstanceConfig* consumerConfig) {
|
367
370
|
|
371
|
+
static void *consumer_recv_msg(void *ptr)
|
372
|
+
{
|
373
|
+
rd_kafka_message_t *ret;
|
374
|
+
HermannInstanceConfig *consumerConfig = (HermannInstanceConfig *) ptr;
|
375
|
+
|
376
|
+
ret = rd_kafka_consume(consumerConfig->rkt, consumerConfig->partition, CONSUMER_RECVMSG_TIMEOUT_MS);
|
377
|
+
|
378
|
+
if ( ret == NULL ) {
|
379
|
+
if ( errno != ETIMEDOUT )
|
380
|
+
fprintf(stderr, "%% Error: %s\n", rd_kafka_err2str( rd_kafka_errno2err(errno)));
|
381
|
+
}
|
382
|
+
|
383
|
+
return (void *) ret;
|
384
|
+
}
|
385
|
+
|
386
|
+
/**
|
387
|
+
* consumer_consume_loop
|
388
|
+
*
|
389
|
+
* A timeout-interrupted loop in which we drop the GVL and attemptto receive
|
390
|
+
* messages from Kafka. We'll check every CONSUMER_RECVMSG_TIMEOUT_MS, or
|
391
|
+
* after every message, to see if the ruby interpreter wants us to exit the
|
392
|
+
* loop.
|
393
|
+
*
|
394
|
+
* @param HermannInstanceConfig* The hermann configuration for this consumer
|
395
|
+
*/
|
396
|
+
|
397
|
+
static void consumer_consume_loop(HermannInstanceConfig* consumerConfig) {
|
398
|
+
rd_kafka_message_t *msg;
|
368
399
|
TRACER("\n");
|
369
400
|
|
370
401
|
while (consumerConfig->run) {
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
402
|
+
#ifdef RB_THREAD_BLOCKING_REGION
|
403
|
+
msg = rb_thread_blocking_region(consumer_recv_msg,
|
404
|
+
consumerConfig,
|
405
|
+
consumer_consume_stop_callback,
|
406
|
+
consumerConfig);
|
407
|
+
#elif HAVE_RB_THREAD_CALL_WITHOUT_GVL
|
408
|
+
msg = rb_thread_call_without_gvl(consumer_recv_msg,
|
409
|
+
consumerConfig,
|
410
|
+
consumer_consume_stop_callback,
|
411
|
+
consumerConfig);
|
412
|
+
#else
|
413
|
+
msg = consumer_recv_msg(consumerConfig);
|
414
|
+
#endif
|
377
415
|
|
416
|
+
if ( msg ) {
|
417
|
+
msg_consume(msg, consumerConfig);
|
418
|
+
rd_kafka_message_destroy(msg);
|
419
|
+
}
|
378
420
|
}
|
379
421
|
}
|
380
422
|
|
@@ -412,23 +454,7 @@ static VALUE consumer_consume(VALUE self, VALUE topic) {
|
|
412
454
|
return Qnil;
|
413
455
|
}
|
414
456
|
|
415
|
-
|
416
|
-
/** The consumer will listen for incoming messages in a loop, timing out and checking the consumerConfig->run
|
417
|
-
* flag every second.
|
418
|
-
*
|
419
|
-
* Call rb_thread_blocking_region to release the GVM lock and allow Ruby to amuse itself while we wait on
|
420
|
-
* IO from Kafka.
|
421
|
-
*
|
422
|
-
* If Ruby needs to interrupt the consumer loop, the stop callback will be invoked and the loop should exit.
|
423
|
-
*/
|
424
|
-
rb_thread_blocking_region(consumer_consume_loop,
|
425
|
-
consumerConfig,
|
426
|
-
consumer_consume_stop_callback,
|
427
|
-
consumerConfig);
|
428
|
-
#else
|
429
|
-
consumer_consume_loop(consumerConfig);
|
430
|
-
#endif
|
431
|
-
|
457
|
+
consumer_consume_loop(consumerConfig);
|
432
458
|
|
433
459
|
/* Stop consuming */
|
434
460
|
rd_kafka_consume_stop(consumerConfig->rkt, consumerConfig->partition);
|
@@ -669,7 +695,7 @@ static VALUE producer_connect(VALUE self, VALUE timeout) {
|
|
669
695
|
&data,
|
670
696
|
timeout_ms);
|
671
697
|
TRACER("err: %s (%i)\n", rd_kafka_err2str(err), err);
|
672
|
-
|
698
|
+
|
673
699
|
if (RD_KAFKA_RESP_ERR_NO_ERROR == err) {
|
674
700
|
TRACER("brokers: %i, topics: %i\n",
|
675
701
|
data->broker_cnt,
|
data/ext/hermann/hermann_lib.h
CHANGED
data/lib/hermann/consumer.rb
CHANGED
@@ -10,30 +10,27 @@ module Hermann
|
|
10
10
|
# Hermann::Consumer provides a simple consumer API which is only safe to be
|
11
11
|
# executed in a single thread
|
12
12
|
class Consumer
|
13
|
-
attr_reader :topic, :
|
13
|
+
attr_reader :topic, :internal
|
14
14
|
|
15
15
|
|
16
16
|
# Instantiate Consumer
|
17
17
|
#
|
18
18
|
# @params [String] kafka topic
|
19
|
-
#
|
20
|
-
# @params [String] group ID
|
21
|
-
#
|
22
|
-
# @params [String] comma separated zookeeper list
|
23
|
-
#
|
24
19
|
# @params [Hash] options for Consumer
|
25
|
-
# @option opts [String]
|
26
|
-
# @option opts [Integer] :partition
|
27
|
-
|
20
|
+
# @option opts [String] :brokers (for MRI) Comma separated list of brokers
|
21
|
+
# @option opts [Integer] :partition (for MRI) The kafka partition
|
22
|
+
# @option opts [Integer] :zookeepers (for jruby) list of zookeeper servers
|
23
|
+
# @option opts [Integer] :group_id (for jruby) client group_id
|
24
|
+
#
|
25
|
+
def initialize(topic, opts = {})
|
28
26
|
@topic = topic
|
29
|
-
@brokers = brokers
|
30
|
-
@partition = partition
|
31
|
-
|
32
27
|
if Hermann.jruby?
|
33
|
-
|
28
|
+
zookeepers, group_id = require_values_at(opts, :zookeepers, :group_id)
|
29
|
+
|
30
|
+
@internal = Hermann::Provider::JavaSimpleConsumer.new(zookeepers, group_id, topic, opts)
|
34
31
|
else
|
35
|
-
brokers
|
36
|
-
|
32
|
+
brokers, partition = require_values_at(opts, :brokers, :partition)
|
33
|
+
|
37
34
|
@internal = Hermann::Lib::Consumer.new(topic, brokers, partition)
|
38
35
|
end
|
39
36
|
end
|
@@ -51,5 +48,12 @@ module Hermann
|
|
51
48
|
#no op
|
52
49
|
end
|
53
50
|
end
|
51
|
+
|
52
|
+
def require_values_at(opts, *args)
|
53
|
+
args.map do |a|
|
54
|
+
raise "Please provide :#{a} option!" unless opts[a]
|
55
|
+
opts.delete(a)
|
56
|
+
end
|
57
|
+
end
|
54
58
|
end
|
55
59
|
end
|
data/lib/hermann/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: hermann
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.23.0.236
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- R. Tyler Croy
|
@@ -10,50 +10,50 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2015-04-
|
13
|
+
date: 2015-04-28 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
|
+
name: concurrent-ruby
|
17
|
+
version_requirements: !ruby/object:Gem::Requirement
|
18
|
+
requirements:
|
19
|
+
- - ~>
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: 0.7.0
|
16
22
|
requirement: !ruby/object:Gem::Requirement
|
17
23
|
requirements:
|
18
24
|
- - ~>
|
19
25
|
- !ruby/object:Gem::Version
|
20
26
|
version: 0.7.0
|
21
|
-
name: concurrent-ruby
|
22
27
|
prerelease: false
|
23
28
|
type: :runtime
|
29
|
+
- !ruby/object:Gem::Dependency
|
30
|
+
name: thread_safe
|
24
31
|
version_requirements: !ruby/object:Gem::Requirement
|
25
32
|
requirements:
|
26
33
|
- - ~>
|
27
34
|
- !ruby/object:Gem::Version
|
28
|
-
version: 0.
|
29
|
-
- !ruby/object:Gem::Dependency
|
35
|
+
version: 0.3.4
|
30
36
|
requirement: !ruby/object:Gem::Requirement
|
31
37
|
requirements:
|
32
38
|
- - ~>
|
33
39
|
- !ruby/object:Gem::Version
|
34
40
|
version: 0.3.4
|
35
|
-
name: thread_safe
|
36
41
|
prerelease: false
|
37
42
|
type: :runtime
|
43
|
+
- !ruby/object:Gem::Dependency
|
44
|
+
name: jar-dependencies
|
38
45
|
version_requirements: !ruby/object:Gem::Requirement
|
39
46
|
requirements:
|
40
47
|
- - ~>
|
41
48
|
- !ruby/object:Gem::Version
|
42
|
-
version: 0.
|
43
|
-
- !ruby/object:Gem::Dependency
|
49
|
+
version: 0.1.9
|
44
50
|
requirement: !ruby/object:Gem::Requirement
|
45
51
|
requirements:
|
46
52
|
- - ~>
|
47
53
|
- !ruby/object:Gem::Version
|
48
54
|
version: 0.1.9
|
49
|
-
name: jar-dependencies
|
50
55
|
prerelease: false
|
51
56
|
type: :runtime
|
52
|
-
version_requirements: !ruby/object:Gem::Requirement
|
53
|
-
requirements:
|
54
|
-
- - ~>
|
55
|
-
- !ruby/object:Gem::Version
|
56
|
-
version: 0.1.9
|
57
57
|
description: Ruby gem for talking to Kafka
|
58
58
|
email:
|
59
59
|
- rtyler.croy@lookout.com
|