hermann 0.23.0.232 → 0.23.0.236
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/ext/hermann/extconf.rb +3 -0
- data/ext/hermann/hermann_lib.c +67 -41
- data/ext/hermann/hermann_lib.h +4 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
NTY5YjNkMzk4YWIwY2Q2MzJmYjYxODBjMmU1YTA1YzYwNDE1YzBlMg==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
NWVlZjNiMTdjOTkzNjljMDRkN2M5NTg4MjY5YTUyNTgwYzMwMzhjNQ==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
N2YyNmJhMTg0YjA4ZmI4ZDlhNWIzYjNhMGU2ZjYzNTVlYzMzZGY1YThhZGRk
|
10
|
+
OGM1MTM2ODk3OTUxZDg1NmQ5NjM5YjNhM2VhMWIxNTk3MzM4MTM4NGNkNWFm
|
11
|
+
MTJkNmJhNzFiNjhkNmYxZGM5NjAxN2Q5MThkNTcwNjBlMzgwMzY=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
NzJkYmI3YzM1MjI4NzUzZWZiYTA2ZTFmNzhjZjE4ZGUxNTU4ZGRmZmEyOThk
|
14
|
+
MTIzMzY4MmIzODYyZTY1YjZjMDZlODM0OWU1OTI5NWMwYTFiZjA5YjU1NjAy
|
15
|
+
YTI5NzlmMWUwNThhMmY0MjRmNjU3Yzk4NWM1MGNkZjMwZTcxNTU=
|
data/ext/hermann/extconf.rb
CHANGED
@@ -146,4 +146,7 @@ dir_config('rdkafka', HEADER_DIRS, LIB_DIRS)
|
|
146
146
|
# <http://blog.zachallett.com/howto-ruby-c-extension-with-a-static-library>
|
147
147
|
$LOCAL_LIBS << File.join(librdkafka.path, 'lib', 'librdkafka.a')
|
148
148
|
|
149
|
+
have_header('ruby/thread.h')
|
150
|
+
have_func('rb_thread_call_without_gvl')
|
151
|
+
|
149
152
|
create_makefile('hermann/hermann_lib')
|
data/ext/hermann/hermann_lib.c
CHANGED
@@ -33,6 +33,11 @@
|
|
33
33
|
|
34
34
|
#include "hermann_lib.h"
|
35
35
|
|
36
|
+
|
37
|
+
/* how long to let librdkafka block on the socket before returning back to the interpreter.
|
38
|
+
* essentially defines how long we wait before consumer_consume_stop_callback() can fire */
|
39
|
+
#define CONSUMER_RECVMSG_TIMEOUT_MS 100
|
40
|
+
|
36
41
|
/**
|
37
42
|
* Convenience function
|
38
43
|
*
|
@@ -204,22 +209,16 @@ static void hexdump(FILE *fp,
|
|
204
209
|
* @param rkmessage rd_kafka_message_t* the message
|
205
210
|
* @param opaque void* opaque context
|
206
211
|
*/
|
207
|
-
static void msg_consume(rd_kafka_message_t *rkmessage,
|
208
|
-
void *opaque) {
|
209
|
-
|
210
|
-
HermannInstanceConfig* cfg;
|
211
|
-
|
212
|
-
cfg = (HermannInstanceConfig*)opaque;
|
213
|
-
|
212
|
+
static void msg_consume(rd_kafka_message_t *rkmessage, HermannInstanceConfig *cfg) {
|
214
213
|
if (rkmessage->err) {
|
215
214
|
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
|
216
|
-
fprintf(stderr,
|
217
|
-
"%% Consumer reached end of %s [%"PRId32"] "
|
218
|
-
"message queue at offset %"PRId64"\n",
|
219
|
-
rd_kafka_topic_name(rkmessage->rkt),
|
220
|
-
rkmessage->partition, rkmessage->offset);
|
221
|
-
|
222
215
|
if (cfg->exit_eof) {
|
216
|
+
fprintf(stderr,
|
217
|
+
"%% Consumer reached end of %s [%"PRId32"] "
|
218
|
+
"message queue at offset %"PRId64"\n",
|
219
|
+
rd_kafka_topic_name(rkmessage->rkt),
|
220
|
+
rkmessage->partition, rkmessage->offset);
|
221
|
+
|
223
222
|
cfg->run = 0;
|
224
223
|
}
|
225
224
|
|
@@ -341,7 +340,7 @@ void consumer_init_kafka(HermannInstanceConfig* config) {
|
|
341
340
|
|
342
341
|
// Ruby gem extensions
|
343
342
|
|
344
|
-
#
|
343
|
+
#if defined(RB_THREAD_BLOCKING_REGION) || defined(HAVE_RB_THREAD_CALL_WITHOUT_GVL)
|
345
344
|
/* NOTE: We only need this method defined if RB_THREAD_BLOCKING_REGION is
|
346
345
|
* defined, otherwise it's unused
|
347
346
|
*/
|
@@ -360,21 +359,64 @@ static void consumer_consume_stop_callback(void *ptr) {
|
|
360
359
|
#endif
|
361
360
|
|
362
361
|
/**
|
363
|
-
*
|
364
|
-
*
|
362
|
+
* consumer_recv_msg
|
363
|
+
*
|
364
|
+
* Consume a single message from the kafka stream. The only function that should be invoked
|
365
|
+
* without the GVL held.
|
366
|
+
*
|
367
|
+
* @param HermannInstanceConfig* The hermann configuration for this consumer
|
368
|
+
*
|
365
369
|
*/
|
366
|
-
void consumer_consume_loop(HermannInstanceConfig* consumerConfig) {
|
367
370
|
|
371
|
+
static void *consumer_recv_msg(void *ptr)
|
372
|
+
{
|
373
|
+
rd_kafka_message_t *ret;
|
374
|
+
HermannInstanceConfig *consumerConfig = (HermannInstanceConfig *) ptr;
|
375
|
+
|
376
|
+
ret = rd_kafka_consume(consumerConfig->rkt, consumerConfig->partition, CONSUMER_RECVMSG_TIMEOUT_MS);
|
377
|
+
|
378
|
+
if ( ret == NULL ) {
|
379
|
+
if ( errno != ETIMEDOUT )
|
380
|
+
fprintf(stderr, "%% Error: %s\n", rd_kafka_err2str( rd_kafka_errno2err(errno)));
|
381
|
+
}
|
382
|
+
|
383
|
+
return (void *) ret;
|
384
|
+
}
|
385
|
+
|
386
|
+
/**
|
387
|
+
* consumer_consume_loop
|
388
|
+
*
|
389
|
+
* A timeout-interrupted loop in which we drop the GVL and attemptto receive
|
390
|
+
* messages from Kafka. We'll check every CONSUMER_RECVMSG_TIMEOUT_MS, or
|
391
|
+
* after every message, to see if the ruby interpreter wants us to exit the
|
392
|
+
* loop.
|
393
|
+
*
|
394
|
+
* @param HermannInstanceConfig* The hermann configuration for this consumer
|
395
|
+
*/
|
396
|
+
|
397
|
+
static void consumer_consume_loop(HermannInstanceConfig* consumerConfig) {
|
398
|
+
rd_kafka_message_t *msg;
|
368
399
|
TRACER("\n");
|
369
400
|
|
370
401
|
while (consumerConfig->run) {
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
402
|
+
#ifdef RB_THREAD_BLOCKING_REGION
|
403
|
+
msg = rb_thread_blocking_region(consumer_recv_msg,
|
404
|
+
consumerConfig,
|
405
|
+
consumer_consume_stop_callback,
|
406
|
+
consumerConfig);
|
407
|
+
#elif HAVE_RB_THREAD_CALL_WITHOUT_GVL
|
408
|
+
msg = rb_thread_call_without_gvl(consumer_recv_msg,
|
409
|
+
consumerConfig,
|
410
|
+
consumer_consume_stop_callback,
|
411
|
+
consumerConfig);
|
412
|
+
#else
|
413
|
+
msg = consumer_recv_msg(consumerConfig);
|
414
|
+
#endif
|
377
415
|
|
416
|
+
if ( msg ) {
|
417
|
+
msg_consume(msg, consumerConfig);
|
418
|
+
rd_kafka_message_destroy(msg);
|
419
|
+
}
|
378
420
|
}
|
379
421
|
}
|
380
422
|
|
@@ -412,23 +454,7 @@ static VALUE consumer_consume(VALUE self, VALUE topic) {
|
|
412
454
|
return Qnil;
|
413
455
|
}
|
414
456
|
|
415
|
-
|
416
|
-
/** The consumer will listen for incoming messages in a loop, timing out and checking the consumerConfig->run
|
417
|
-
* flag every second.
|
418
|
-
*
|
419
|
-
* Call rb_thread_blocking_region to release the GVM lock and allow Ruby to amuse itself while we wait on
|
420
|
-
* IO from Kafka.
|
421
|
-
*
|
422
|
-
* If Ruby needs to interrupt the consumer loop, the stop callback will be invoked and the loop should exit.
|
423
|
-
*/
|
424
|
-
rb_thread_blocking_region(consumer_consume_loop,
|
425
|
-
consumerConfig,
|
426
|
-
consumer_consume_stop_callback,
|
427
|
-
consumerConfig);
|
428
|
-
#else
|
429
|
-
consumer_consume_loop(consumerConfig);
|
430
|
-
#endif
|
431
|
-
|
457
|
+
consumer_consume_loop(consumerConfig);
|
432
458
|
|
433
459
|
/* Stop consuming */
|
434
460
|
rd_kafka_consume_stop(consumerConfig->rkt, consumerConfig->partition);
|
@@ -669,7 +695,7 @@ static VALUE producer_connect(VALUE self, VALUE timeout) {
|
|
669
695
|
&data,
|
670
696
|
timeout_ms);
|
671
697
|
TRACER("err: %s (%i)\n", rd_kafka_err2str(err), err);
|
672
|
-
|
698
|
+
|
673
699
|
if (RD_KAFKA_RESP_ERR_NO_ERROR == err) {
|
674
700
|
TRACER("brokers: %i, topics: %i\n",
|
675
701
|
data->broker_cnt,
|
data/ext/hermann/hermann_lib.h
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: hermann
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.23.0.
|
4
|
+
version: 0.23.0.236
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- R. Tyler Croy
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2015-04-
|
13
|
+
date: 2015-04-28 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: concurrent-ruby
|
@@ -102,7 +102,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
102
102
|
version: '0'
|
103
103
|
requirements: []
|
104
104
|
rubyforge_project:
|
105
|
-
rubygems_version: 2.4.
|
105
|
+
rubygems_version: 2.4.5
|
106
106
|
signing_key:
|
107
107
|
specification_version: 3
|
108
108
|
summary: A Kafka consumer/producer gem supporting both MRI and JRuby
|