hermann 0.11-x86-darwin-13
Sign up to get free protection for your applications and to get access to all the features.
- data/Rakefile +31 -0
- data/bin/hermann +4 -0
- data/ext/extconf.rb +32 -0
- data/ext/hermann_lib.c +727 -0
- data/ext/hermann_lib.h +83 -0
- data/ext/hermann_lib.o +0 -0
- data/lib/hermann.rb +1 -0
- data/test/test_hermann.rb +9 -0
- metadata +74 -0
data/Rakefile
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
|
2
|
+
require 'rake/clean'
|
3
|
+
|
4
|
+
EXT_CONF = "ext/extconf.rb"
|
5
|
+
MAKEFILE = 'ext/Makefile'
|
6
|
+
MODULE = 'ext/hermann_lib.so'
|
7
|
+
SRC = Dir.glob('ext/*.c')
|
8
|
+
SRC << MAKEFILE
|
9
|
+
|
10
|
+
CLEAN.include [ 'ext/*.o', 'ext/depend', 'ext/hermann_lib.bundle', MODULE ]
|
11
|
+
CLOBBER.include [ 'config.save', 'ext/mkmf.log', 'ext/hermann_lib.bundle', MAKEFILE ]
|
12
|
+
|
13
|
+
file MAKEFILE => EXT_CONF do |t|
|
14
|
+
Dir::chdir(File::dirname(EXT_CONF)) do
|
15
|
+
unless sh "ruby #{File::basename(EXT_CONF)}"
|
16
|
+
$stderr.puts "Failed to run extconf"
|
17
|
+
break
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
file MODULE => SRC do |t|
|
23
|
+
Dir::chdir(File::dirname(EXT_CONF)) do
|
24
|
+
unless sh "make"
|
25
|
+
$stderr.puts "make failed"
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
desc "Build the native library"
|
31
|
+
task :build => MODULE
|
data/bin/hermann
ADDED
data/ext/extconf.rb
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
# External configuration for Hermann Gem
|
2
|
+
|
3
|
+
require 'mkmf'
|
4
|
+
|
5
|
+
RbConfig::MAKEFILE_CONFIG['CC'] = ENV['CC'] if ENV['CC']
|
6
|
+
|
7
|
+
LIBDIR = RbConfig::CONFIG['libdir']
|
8
|
+
INCLUDEDIR = RbConfig::CONFIG['includedir']
|
9
|
+
|
10
|
+
puts "Library Dir: #{LIBDIR}\n"
|
11
|
+
puts "Include Dir: #{INCLUDEDIR}"
|
12
|
+
|
13
|
+
HEADER_DIRS = [INCLUDEDIR]
|
14
|
+
|
15
|
+
LIB_DIRS = [LIBDIR]
|
16
|
+
|
17
|
+
dir_config('rdkafka', HEADER_DIRS, LIB_DIRS)
|
18
|
+
|
19
|
+
unless find_header('librdkafka/rdkafka.h')
|
20
|
+
abort "librdkafka not installed"
|
21
|
+
end
|
22
|
+
|
23
|
+
unless find_library('rdkafka', 'rd_kafka_conf_new')
|
24
|
+
abort "librdkafka not installed"
|
25
|
+
end
|
26
|
+
|
27
|
+
#unless have_func "rb_thread_blocking_region"
|
28
|
+
# abort "rb_thread_blocking_region function missing"
|
29
|
+
#end
|
30
|
+
|
31
|
+
# create_header('hermann_lib.h')
|
32
|
+
create_makefile('hermann/hermann_lib')
|
data/ext/hermann_lib.c
ADDED
@@ -0,0 +1,727 @@
|
|
1
|
+
/*
|
2
|
+
* hermann_lib.c - Ruby wrapper for the librdkafka library
|
3
|
+
*
|
4
|
+
* Copyright (c) 2014 Stan Campbell
|
5
|
+
* All rights reserved.
|
6
|
+
*
|
7
|
+
* Redistribution and use in source and binary forms, with or without
|
8
|
+
* modification, are permitted provided that the following conditions are met:
|
9
|
+
*
|
10
|
+
* 1. Redistributions of source code must retain the above copyright notice,
|
11
|
+
* this list of conditions and the following disclaimer.
|
12
|
+
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
13
|
+
* this list of conditions and the following disclaimer in the documentation
|
14
|
+
* and/or other materials provided with the distribution.
|
15
|
+
*
|
16
|
+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
17
|
+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
18
|
+
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
19
|
+
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
20
|
+
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
21
|
+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
22
|
+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
23
|
+
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
24
|
+
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
25
|
+
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
26
|
+
* POSSIBILITY OF SUCH DAMAGE.
|
27
|
+
*/
|
28
|
+
|
29
|
+
/* Much of the librdkafka library calls were lifted from rdkafka_example.c */
|
30
|
+
|
31
|
+
#include "hermann_lib.h"
|
32
|
+
|
33
|
+
/**
|
34
|
+
* Utility functions
|
35
|
+
*/
|
36
|
+
|
37
|
+
|
38
|
+
/**
|
39
|
+
* Convenience function
|
40
|
+
*
|
41
|
+
* @param msg char* the string to be logged under debugging.
|
42
|
+
*/
|
43
|
+
void log_debug(char* msg) {
|
44
|
+
if(DEBUG) {
|
45
|
+
fprintf(stderr, "%s\n", msg);
|
46
|
+
}
|
47
|
+
}
|
48
|
+
|
49
|
+
/**
|
50
|
+
* Message delivery report callback.
|
51
|
+
* Called once for each message.
|
52
|
+
*
|
53
|
+
* @param rk rd_kafka_t* instance of producer or consumer
|
54
|
+
* @param payload void* the payload of the message
|
55
|
+
* @param len size_t the length of the payload in bytes
|
56
|
+
* @param error_code int
|
57
|
+
* @param opaque void* optional context
|
58
|
+
* @param msg_opaque void* it's opaque
|
59
|
+
*/
|
60
|
+
static void msg_delivered (rd_kafka_t *rk,
|
61
|
+
void *payload, size_t len,
|
62
|
+
int error_code,
|
63
|
+
void *opaque, void *msg_opaque) {
|
64
|
+
|
65
|
+
if (error_code)
|
66
|
+
fprintf(stderr, "%% Message delivery failed: %s\n",
|
67
|
+
rd_kafka_err2str(error_code));
|
68
|
+
}
|
69
|
+
|
70
|
+
/**
|
71
|
+
* Producer partitioner callback.
|
72
|
+
* Used to determine the target partition within a topic for production.
|
73
|
+
*
|
74
|
+
* Returns an integer partition number or RD_KAFKA_PARTITION_UA if no
|
75
|
+
* available partition could be determined.
|
76
|
+
*
|
77
|
+
* @param rkt rd_kafka_topic_t* the topic
|
78
|
+
* @param keydata void* key information for calculating the partition
|
79
|
+
* @param keylen size_t key size
|
80
|
+
* @param partition_cnt int32_t the count of the number of partitions
|
81
|
+
* @param rkt_opaque void* opaque topic info
|
82
|
+
* @param msg_opaque void* opaque message info
|
83
|
+
*/
|
84
|
+
static int32_t producer_paritioner_callback( const rd_kafka_topic_t *rkt,
|
85
|
+
const void *keydata,
|
86
|
+
size_t keylen,
|
87
|
+
int32_t partition_cnt,
|
88
|
+
void *rkt_opaque,
|
89
|
+
void *msg_opaque) {
|
90
|
+
/* Pick a random partition */
|
91
|
+
int retry;
|
92
|
+
for(retry=0;retry<partition_cnt;retry++) {
|
93
|
+
int32_t partition = rand() % partition_cnt;
|
94
|
+
if(rd_kafka_topic_partition_available(rkt, partition)) {
|
95
|
+
break; /* this one will do */
|
96
|
+
}
|
97
|
+
}
|
98
|
+
}
|
99
|
+
|
100
|
+
/**
|
101
|
+
* hexdump
|
102
|
+
*
|
103
|
+
* Write the given payload to file in hex notation.
|
104
|
+
*
|
105
|
+
* @param fp FILE* the file into which to write
|
106
|
+
* @param name char* name
|
107
|
+
* @param ptr void* payload
|
108
|
+
* @param len size_t payload length
|
109
|
+
*/
|
110
|
+
static void hexdump (FILE *fp, const char *name, const void *ptr, size_t len) {
|
111
|
+
const char *p = (const char *)ptr;
|
112
|
+
int of = 0;
|
113
|
+
|
114
|
+
|
115
|
+
if (name)
|
116
|
+
fprintf(fp, "%s hexdump (%zd bytes):\n", name, len);
|
117
|
+
|
118
|
+
for (of = 0 ; of < len ; of += 16) {
|
119
|
+
char hexen[16*3+1];
|
120
|
+
char charen[16+1];
|
121
|
+
int hof = 0;
|
122
|
+
|
123
|
+
int cof = 0;
|
124
|
+
int i;
|
125
|
+
|
126
|
+
for (i = of ; i < of + 16 && i < len ; i++) {
|
127
|
+
hof += sprintf(hexen+hof, "%02x ", p[i] & 0xff);
|
128
|
+
cof += sprintf(charen+cof, "%c",
|
129
|
+
isprint((int)p[i]) ? p[i] : '.');
|
130
|
+
}
|
131
|
+
fprintf(fp, "%08x: %-48s %-16s\n",
|
132
|
+
of, hexen, charen);
|
133
|
+
}
|
134
|
+
}
|
135
|
+
|
136
|
+
/**
|
137
|
+
* msg_consume
|
138
|
+
*
|
139
|
+
* Callback on message receipt.
|
140
|
+
*
|
141
|
+
* @param rkmessage rd_kafka_message_t* the message
|
142
|
+
* @param opaque void* opaque context
|
143
|
+
*/
|
144
|
+
static void msg_consume (rd_kafka_message_t *rkmessage,
|
145
|
+
void *opaque) {
|
146
|
+
|
147
|
+
HermannInstanceConfig* cfg;
|
148
|
+
|
149
|
+
cfg = (HermannInstanceConfig*)opaque;
|
150
|
+
|
151
|
+
if (rkmessage->err) {
|
152
|
+
if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
|
153
|
+
fprintf(stderr,
|
154
|
+
"%% Consumer reached end of %s [%"PRId32"] "
|
155
|
+
"message queue at offset %"PRId64"\n",
|
156
|
+
rd_kafka_topic_name(rkmessage->rkt),
|
157
|
+
rkmessage->partition, rkmessage->offset);
|
158
|
+
|
159
|
+
if (cfg->exit_eof)
|
160
|
+
cfg->run = 0;
|
161
|
+
|
162
|
+
return;
|
163
|
+
}
|
164
|
+
|
165
|
+
fprintf(stderr, "%% Consume error for topic \"%s\" [%"PRId32"] "
|
166
|
+
"offset %"PRId64": %s\n",
|
167
|
+
rd_kafka_topic_name(rkmessage->rkt),
|
168
|
+
rkmessage->partition,
|
169
|
+
rkmessage->offset,
|
170
|
+
rd_kafka_message_errstr(rkmessage));
|
171
|
+
return;
|
172
|
+
}
|
173
|
+
|
174
|
+
if (DEBUG && rkmessage->key_len) {
|
175
|
+
if (output == OUTPUT_HEXDUMP)
|
176
|
+
hexdump(stdout, "Message Key",
|
177
|
+
rkmessage->key, rkmessage->key_len);
|
178
|
+
else
|
179
|
+
printf("Key: %.*s\n",
|
180
|
+
(int)rkmessage->key_len, (char *)rkmessage->key);
|
181
|
+
}
|
182
|
+
|
183
|
+
if (output == OUTPUT_HEXDUMP) {
|
184
|
+
if(DEBUG)
|
185
|
+
hexdump(stdout, "Message Payload", rkmessage->payload, rkmessage->len);
|
186
|
+
} else {
|
187
|
+
if(DEBUG)
|
188
|
+
printf("%.*s\n", (int)rkmessage->len, (char *)rkmessage->payload);
|
189
|
+
}
|
190
|
+
|
191
|
+
// Yield the data to the Consumer's block
|
192
|
+
if(rb_block_given_p()) {
|
193
|
+
VALUE value = rb_str_new((char *)rkmessage->payload, rkmessage->len);
|
194
|
+
rb_yield(value);
|
195
|
+
} else {
|
196
|
+
if(DEBUG)
|
197
|
+
fprintf(stderr, "No block given\n"); // todo: should this be an error?
|
198
|
+
}
|
199
|
+
}
|
200
|
+
|
201
|
+
/**
|
202
|
+
* logger
|
203
|
+
*
|
204
|
+
* Kafka logger callback (optional)
|
205
|
+
*
|
206
|
+
* todo: introduce better logging
|
207
|
+
*
|
208
|
+
* @param rk rd_kafka_t the producer or consumer
|
209
|
+
* @param level int the log level
|
210
|
+
* @param fac char* something of which I am unaware
|
211
|
+
* @param buf char* the log message
|
212
|
+
*/
|
213
|
+
static void logger (const rd_kafka_t *rk, int level,
|
214
|
+
const char *fac, const char *buf) {
|
215
|
+
struct timeval tv;
|
216
|
+
gettimeofday(&tv, NULL);
|
217
|
+
fprintf(stderr, "%u.%03u RDKAFKA-%i-%s: %s: %s\n",
|
218
|
+
(int)tv.tv_sec, (int)(tv.tv_usec / 1000),
|
219
|
+
level, fac, rd_kafka_name(rk), buf);
|
220
|
+
}
|
221
|
+
|
222
|
+
/**
|
223
|
+
* consumer_init_kafka
|
224
|
+
*
|
225
|
+
* Initialize the Kafka context and instantiate a consumer.
|
226
|
+
*
|
227
|
+
* @param config HermannInstanceConfig* pointer to the instance configuration for this producer or consumer
|
228
|
+
*/
|
229
|
+
void consumer_init_kafka(HermannInstanceConfig* config) {
|
230
|
+
|
231
|
+
config->quiet = !isatty(STDIN_FILENO);
|
232
|
+
|
233
|
+
/* Kafka configuration */
|
234
|
+
config->conf = rd_kafka_conf_new();
|
235
|
+
|
236
|
+
/* Topic configuration */
|
237
|
+
config->topic_conf = rd_kafka_topic_conf_new();
|
238
|
+
|
239
|
+
/* Create Kafka handle */
|
240
|
+
if (!(config->rk = rd_kafka_new(RD_KAFKA_CONSUMER, config->conf,
|
241
|
+
config->errstr, sizeof(config->errstr)))) {
|
242
|
+
fprintf(stderr, "%% Failed to create new consumer: %s\n", config->errstr);
|
243
|
+
exit(1);
|
244
|
+
}
|
245
|
+
|
246
|
+
/* Set logger */
|
247
|
+
rd_kafka_set_logger(config->rk, logger);
|
248
|
+
rd_kafka_set_log_level(config->rk, LOG_DEBUG);
|
249
|
+
|
250
|
+
/* TODO: offset calculation */
|
251
|
+
config->start_offset = RD_KAFKA_OFFSET_END;
|
252
|
+
|
253
|
+
/* Add brokers */
|
254
|
+
if(rd_kafka_brokers_add(config->rk, config->brokers) == 0) {
|
255
|
+
fprintf(stderr, "%% No valid brokers specified\n");
|
256
|
+
exit(1);
|
257
|
+
}
|
258
|
+
|
259
|
+
/* Create topic */
|
260
|
+
config->rkt = rd_kafka_topic_new(config->rk, config->topic, config->topic_conf);
|
261
|
+
|
262
|
+
/* We're now initialized */
|
263
|
+
config->isInitialized = 1;
|
264
|
+
}
|
265
|
+
|
266
|
+
// Ruby gem extensions
|
267
|
+
|
268
|
+
/**
|
269
|
+
* Callback invoked if Ruby needs to stop our Consumer's IO loop for any reason (system exit, etc.)
|
270
|
+
*/
|
271
|
+
static void consumer_consume_stop_callback(void *ptr) {
|
272
|
+
HermannInstanceConfig* config = (HermannInstanceConfig*)ptr;
|
273
|
+
|
274
|
+
config->run = 0;
|
275
|
+
}
|
276
|
+
|
277
|
+
/**
|
278
|
+
* Loop on a timeout to receive messages from Kafka. When the consumer_consume_stop_callback is invoked by Ruby,
|
279
|
+
* we'll break out of our loop and return.
|
280
|
+
*/
|
281
|
+
void consumer_consume_loop(HermannInstanceConfig* consumerConfig) {
|
282
|
+
|
283
|
+
while (consumerConfig->run) {
|
284
|
+
rd_kafka_message_t *rkmessage;
|
285
|
+
|
286
|
+
if(rd_kafka_consume_callback(consumerConfig->rkt, consumerConfig->partition,
|
287
|
+
1000/*timeout*/,
|
288
|
+
msg_consume,
|
289
|
+
consumerConfig) < 0) {
|
290
|
+
fprintf(stderr, "%% Error: %s\n", rd_kafka_err2str( rd_kafka_errno2err(errno)));
|
291
|
+
}
|
292
|
+
|
293
|
+
}
|
294
|
+
}
|
295
|
+
|
296
|
+
/**
|
297
|
+
* Hermann::Consumer.consume
|
298
|
+
*
|
299
|
+
* Begin listening on the configured topic for messages. msg_consume will be called on each message received.
|
300
|
+
*
|
301
|
+
* @param VALUE self the Ruby object for this consumer
|
302
|
+
*/
|
303
|
+
static VALUE consumer_consume(VALUE self) {
|
304
|
+
|
305
|
+
HermannInstanceConfig* consumerConfig;
|
306
|
+
|
307
|
+
Data_Get_Struct(self, HermannInstanceConfig, consumerConfig);
|
308
|
+
|
309
|
+
if(consumerConfig->topic==NULL) {
|
310
|
+
fprintf(stderr, "Topic is null!");
|
311
|
+
return;
|
312
|
+
}
|
313
|
+
|
314
|
+
if(!consumerConfig->isInitialized) {
|
315
|
+
consumer_init_kafka(consumerConfig);
|
316
|
+
}
|
317
|
+
|
318
|
+
/* Start consuming */
|
319
|
+
if (rd_kafka_consume_start(consumerConfig->rkt, consumerConfig->partition, consumerConfig->start_offset) == -1){
|
320
|
+
fprintf(stderr, "%% Failed to start consuming: %s\n",
|
321
|
+
rd_kafka_err2str(rd_kafka_errno2err(errno)));
|
322
|
+
exit(1);
|
323
|
+
}
|
324
|
+
|
325
|
+
#ifdef RB_THREAD_BLOCKING_REGION
|
326
|
+
/** The consumer will listen for incoming messages in a loop, timing out and checking the consumerConfig->run
|
327
|
+
* flag every second.
|
328
|
+
*
|
329
|
+
* Call rb_thread_blocking_region to release the GVM lock and allow Ruby to amuse itself while we wait on
|
330
|
+
* IO from Kafka.
|
331
|
+
*
|
332
|
+
* If Ruby needs to interrupt the consumer loop, the stop callback will be invoked and the loop should exit.
|
333
|
+
*/
|
334
|
+
rb_thread_blocking_region(consumer_consume_loop, consumerConfig, consumer_consume_stop_callback, consumerConfig);
|
335
|
+
#else
|
336
|
+
consumer_consume_loop(consumerConfig);
|
337
|
+
#endif
|
338
|
+
|
339
|
+
|
340
|
+
/* Stop consuming */
|
341
|
+
rd_kafka_consume_stop(consumerConfig->rkt, consumerConfig->partition);
|
342
|
+
|
343
|
+
return Qnil;
|
344
|
+
}
|
345
|
+
|
346
|
+
/**
|
347
|
+
* producer_init_kafka
|
348
|
+
*
|
349
|
+
* Initialize the producer instance, setting up the Kafka topic and context.
|
350
|
+
*
|
351
|
+
* @param config HermannInstanceConfig* the instance configuration associated with this producer.
|
352
|
+
*/
|
353
|
+
void producer_init_kafka(HermannInstanceConfig* config) {
|
354
|
+
|
355
|
+
config->quiet = !isatty(STDIN_FILENO);
|
356
|
+
|
357
|
+
/* Kafka configuration */
|
358
|
+
config->conf = rd_kafka_conf_new();
|
359
|
+
|
360
|
+
/* Topic configuration */
|
361
|
+
config->topic_conf = rd_kafka_topic_conf_new();
|
362
|
+
|
363
|
+
/* Set up a message delivery report callback.
|
364
|
+
* It will be called once for each message, either on successful
|
365
|
+
* delivery to broker, or upon failure to deliver to broker. */
|
366
|
+
rd_kafka_conf_set_dr_cb(config->conf, msg_delivered);
|
367
|
+
|
368
|
+
/* Create Kafka handle */
|
369
|
+
if (!(config->rk = rd_kafka_new(RD_KAFKA_PRODUCER, config->conf, config->errstr, sizeof(config->errstr)))) {
|
370
|
+
fprintf(stderr,
|
371
|
+
"%% Failed to create new producer: %s\n", config->errstr);
|
372
|
+
exit(1);
|
373
|
+
}
|
374
|
+
|
375
|
+
/* Set logger */
|
376
|
+
rd_kafka_set_logger(config->rk, logger);
|
377
|
+
rd_kafka_set_log_level(config->rk, LOG_DEBUG);
|
378
|
+
|
379
|
+
if(rd_kafka_brokers_add(config->rk, config->brokers) == 0) {
|
380
|
+
fprintf(stderr, "%% No valid brokers specified\n");
|
381
|
+
exit(1);
|
382
|
+
}
|
383
|
+
|
384
|
+
/* Create topic */
|
385
|
+
config->rkt = rd_kafka_topic_new(config->rk, config->topic, config->topic_conf);
|
386
|
+
|
387
|
+
/* Set the partitioner callback */
|
388
|
+
rd_kafka_topic_conf_set_partitioner_cb( config->topic_conf, producer_paritioner_callback );
|
389
|
+
|
390
|
+
/* We're now initialized */
|
391
|
+
config->isInitialized = 1;
|
392
|
+
}
|
393
|
+
|
394
|
+
/**
|
395
|
+
* producer_push_single
|
396
|
+
*
|
397
|
+
* @param self VALUE the Ruby producer instance
|
398
|
+
* @param message VALUE the ruby String containing the outgoing message.
|
399
|
+
*/
|
400
|
+
static VALUE producer_push_single(VALUE self, VALUE message) {
|
401
|
+
|
402
|
+
HermannInstanceConfig* producerConfig;
|
403
|
+
char buf[2048];
|
404
|
+
|
405
|
+
Data_Get_Struct(self, HermannInstanceConfig, producerConfig);
|
406
|
+
|
407
|
+
if(producerConfig->topic==NULL) {
|
408
|
+
fprintf(stderr, "Topic is null!");
|
409
|
+
return self;
|
410
|
+
}
|
411
|
+
|
412
|
+
if(!producerConfig->isInitialized) {
|
413
|
+
producer_init_kafka(producerConfig);
|
414
|
+
}
|
415
|
+
|
416
|
+
char *msg = StringValueCStr(message);
|
417
|
+
strcpy(buf, msg);
|
418
|
+
|
419
|
+
size_t len = strlen(buf);
|
420
|
+
if (buf[len-1] == '\n')
|
421
|
+
buf[--len] = '\0';
|
422
|
+
|
423
|
+
/* Send/Produce message. */
|
424
|
+
if (rd_kafka_produce(producerConfig->rkt, producerConfig->partition, RD_KAFKA_MSG_F_COPY,
|
425
|
+
/* Payload and length */
|
426
|
+
buf, len,
|
427
|
+
/* Optional key and its length */
|
428
|
+
NULL, 0,
|
429
|
+
/* Message opaque, provided in
|
430
|
+
* delivery report callback as
|
431
|
+
* msg_opaque. */
|
432
|
+
NULL) == -1) {
|
433
|
+
|
434
|
+
fprintf(stderr, "%% Failed to produce to topic %s partition %i: %s\n",
|
435
|
+
rd_kafka_topic_name(producerConfig->rkt), producerConfig->partition,
|
436
|
+
rd_kafka_err2str(rd_kafka_errno2err(errno)));
|
437
|
+
|
438
|
+
/* Poll to handle delivery reports */
|
439
|
+
rd_kafka_poll(producerConfig->rk, 10);
|
440
|
+
}
|
441
|
+
|
442
|
+
/* Must poll to handle delivery reports */
|
443
|
+
rd_kafka_poll(producerConfig->rk, 0);
|
444
|
+
|
445
|
+
return self;
|
446
|
+
}
|
447
|
+
|
448
|
+
/**
|
449
|
+
* producer_push_array
|
450
|
+
*
|
451
|
+
* Publish each of the messages in array on the configured topic.
|
452
|
+
*
|
453
|
+
* @param self VALUE the instance of the Ruby Producer object
|
454
|
+
* @param length int the length of the outgoing messages array
|
455
|
+
* @param array VALUE the Ruby array of messages
|
456
|
+
*/
|
457
|
+
static VALUE producer_push_array(VALUE self, int length, VALUE array) {
|
458
|
+
|
459
|
+
int i;
|
460
|
+
VALUE message;
|
461
|
+
|
462
|
+
for(i=0;i<length;i++) {
|
463
|
+
message = RARRAY_PTR(array)[i];
|
464
|
+
producer_push_single(self, message);
|
465
|
+
}
|
466
|
+
|
467
|
+
return self;
|
468
|
+
}
|
469
|
+
|
470
|
+
/**
|
471
|
+
* Hermann::Producer.push(msg)
|
472
|
+
*
|
473
|
+
* Publish the given message on the configured topic.
|
474
|
+
*
|
475
|
+
* @param self VALUE the Ruby instance of the Producer.
|
476
|
+
* @param message VALUE the Ruby string containing the message.
|
477
|
+
*/
|
478
|
+
static VALUE producer_push(VALUE self, VALUE message) {
|
479
|
+
|
480
|
+
VALUE arrayP = rb_check_array_type(message);
|
481
|
+
|
482
|
+
if(!NIL_P(arrayP)) {
|
483
|
+
return producer_push_array(self, RARRAY_LEN(arrayP), message);
|
484
|
+
} else {
|
485
|
+
return producer_push_single(self, message);
|
486
|
+
}
|
487
|
+
}
|
488
|
+
|
489
|
+
/**
|
490
|
+
* consumer_free
|
491
|
+
*
|
492
|
+
* Callback called when Ruby needs to GC the configuration associated with an Hermann instance.
|
493
|
+
*
|
494
|
+
* @param p void* the instance of an HermannInstanceConfig to be freed from allocated memory.
|
495
|
+
*/
|
496
|
+
static void consumer_free(void * p) {
|
497
|
+
|
498
|
+
HermannInstanceConfig* config = (HermannInstanceConfig *)p;
|
499
|
+
|
500
|
+
// the p *should* contain a pointer to the consumerConfig which also must be freed
|
501
|
+
rd_kafka_topic_destroy(config->rkt);
|
502
|
+
|
503
|
+
rd_kafka_destroy(config->rk);
|
504
|
+
|
505
|
+
// clean up the struct
|
506
|
+
free(config);
|
507
|
+
}
|
508
|
+
|
509
|
+
/**
|
510
|
+
* consumer_allocate
|
511
|
+
*
|
512
|
+
* Allocate and wrap an HermannInstanceConfig for this Consumer object.
|
513
|
+
*
|
514
|
+
* @param klass VALUE the class of the enclosing Ruby object.
|
515
|
+
*/
|
516
|
+
static VALUE consumer_allocate(VALUE klass) {
|
517
|
+
|
518
|
+
VALUE obj;
|
519
|
+
|
520
|
+
HermannInstanceConfig* consumerConfig = ALLOC(HermannInstanceConfig);
|
521
|
+
obj = Data_Wrap_Struct(klass, 0, consumer_free, consumerConfig);
|
522
|
+
|
523
|
+
return obj;
|
524
|
+
}
|
525
|
+
|
526
|
+
/**
|
527
|
+
* consumer_initialize
|
528
|
+
*
|
529
|
+
* todo: configure the brokers through passed parameter, later through zk
|
530
|
+
*
|
531
|
+
* Set up the Consumer's HermannInstanceConfig context.
|
532
|
+
*
|
533
|
+
* @param self VALUE the Ruby instance of the Consumer
|
534
|
+
* @param topic VALUE a Ruby string
|
535
|
+
* @param brokers VALUE a Ruby string containing list of host:port
|
536
|
+
* @param partition VALUE a Ruby number
|
537
|
+
*/
|
538
|
+
static VALUE consumer_initialize(VALUE self, VALUE topic, VALUE brokers, VALUE partition) {
|
539
|
+
|
540
|
+
HermannInstanceConfig* consumerConfig;
|
541
|
+
char* topicPtr;
|
542
|
+
char* brokersPtr;
|
543
|
+
int partitionNo;
|
544
|
+
|
545
|
+
topicPtr = StringValuePtr(topic);
|
546
|
+
brokersPtr = StringValuePtr(brokers);
|
547
|
+
partitionNo = FIX2INT(partition);
|
548
|
+
Data_Get_Struct(self, HermannInstanceConfig, consumerConfig);
|
549
|
+
|
550
|
+
consumerConfig->topic = topicPtr;
|
551
|
+
consumerConfig->brokers = brokersPtr;
|
552
|
+
consumerConfig->partition = partitionNo;
|
553
|
+
consumerConfig->run = 1;
|
554
|
+
consumerConfig->exit_eof = 0;
|
555
|
+
consumerConfig->quiet = 0;
|
556
|
+
|
557
|
+
return self;
|
558
|
+
}
|
559
|
+
|
560
|
+
/**
|
561
|
+
* consumer_init_copy
|
562
|
+
*
|
563
|
+
* When copying into a new instance of a Consumer, reproduce the configuration info.
|
564
|
+
*
|
565
|
+
* @param copy VALUE the Ruby Consumer instance (with configuration) as destination
|
566
|
+
* @param orig VALUE the Ruby Consumer instance (with configuration) as source
|
567
|
+
*
|
568
|
+
*/
|
569
|
+
static VALUE consumer_init_copy(VALUE copy, VALUE orig) {
|
570
|
+
HermannInstanceConfig* orig_config;
|
571
|
+
HermannInstanceConfig* copy_config;
|
572
|
+
|
573
|
+
if(copy == orig) {
|
574
|
+
return copy;
|
575
|
+
}
|
576
|
+
|
577
|
+
if (TYPE(orig) != T_DATA || RDATA(orig)->dfree != (RUBY_DATA_FUNC)consumer_free) {
|
578
|
+
rb_raise(rb_eTypeError, "wrong argument type");
|
579
|
+
}
|
580
|
+
|
581
|
+
Data_Get_Struct(orig, HermannInstanceConfig, orig_config);
|
582
|
+
Data_Get_Struct(copy, HermannInstanceConfig, copy_config);
|
583
|
+
|
584
|
+
// Copy over the data from one struct to the other
|
585
|
+
MEMCPY(copy_config, orig_config, HermannInstanceConfig, 1);
|
586
|
+
|
587
|
+
return copy;
|
588
|
+
}
|
589
|
+
|
590
|
+
/**
|
591
|
+
* producer_free
|
592
|
+
*
|
593
|
+
* Reclaim memory allocated to the Producer's configuration
|
594
|
+
*
|
595
|
+
* @param p void* the instance's configuration struct
|
596
|
+
*/
|
597
|
+
static void producer_free(void * p) {
|
598
|
+
|
599
|
+
HermannInstanceConfig* config = (HermannInstanceConfig *)p;
|
600
|
+
|
601
|
+
// Clean up the topic
|
602
|
+
rd_kafka_topic_destroy(config->rkt);
|
603
|
+
|
604
|
+
// Take care of the producer instance
|
605
|
+
rd_kafka_destroy(config->rk);
|
606
|
+
|
607
|
+
// Free the struct
|
608
|
+
free(config);
|
609
|
+
}
|
610
|
+
|
611
|
+
/**
|
612
|
+
* producer_allocate
|
613
|
+
*
|
614
|
+
* Allocate the memory for a Producer's configuration
|
615
|
+
*
|
616
|
+
* @param klass VALUE the class of the Producer
|
617
|
+
*/
|
618
|
+
static VALUE producer_allocate(VALUE klass) {
|
619
|
+
|
620
|
+
VALUE obj;
|
621
|
+
|
622
|
+
HermannInstanceConfig* producerConfig = ALLOC(HermannInstanceConfig);
|
623
|
+
obj = Data_Wrap_Struct(klass, 0, producer_free, producerConfig);
|
624
|
+
|
625
|
+
return obj;
|
626
|
+
}
|
627
|
+
|
628
|
+
/**
|
629
|
+
* producer_initialize
|
630
|
+
*
|
631
|
+
* Set up the configuration context for the Producer instance
|
632
|
+
*
|
633
|
+
* @param self VALUE the Producer instance
|
634
|
+
* @param topic VALUE the Ruby string naming the topic
|
635
|
+
* @param brokers VALUE a Ruby string containing host:port pairs separated by commas
|
636
|
+
*/
|
637
|
+
static VALUE producer_initialize(VALUE self, VALUE topic, VALUE brokers) {
|
638
|
+
|
639
|
+
HermannInstanceConfig* producerConfig;
|
640
|
+
char* topicPtr;
|
641
|
+
char* brokersPtr;
|
642
|
+
|
643
|
+
topicPtr = StringValuePtr(topic);
|
644
|
+
brokersPtr = StringValuePtr(brokers);
|
645
|
+
Data_Get_Struct(self, HermannInstanceConfig, producerConfig);
|
646
|
+
|
647
|
+
producerConfig->topic = topicPtr;
|
648
|
+
producerConfig->brokers = brokersPtr;
|
649
|
+
/** Using RD_KAFKA_PARTITION_UA specifies we want the partitioner callback to be called to determine the target
|
650
|
+
* partition
|
651
|
+
*/
|
652
|
+
producerConfig->partition = RD_KAFKA_PARTITION_UA;
|
653
|
+
producerConfig->run = 1;
|
654
|
+
producerConfig->exit_eof = 0;
|
655
|
+
producerConfig->quiet = 0;
|
656
|
+
|
657
|
+
return self;
|
658
|
+
}
|
659
|
+
|
660
|
+
/**
|
661
|
+
* producer_init_copy
|
662
|
+
*
|
663
|
+
* Copy the configuration information from orig into copy for the given Producer instances.
|
664
|
+
*
|
665
|
+
* @param copy VALUE destination Producer
|
666
|
+
* @param orign VALUE source Producer
|
667
|
+
*/
|
668
|
+
static VALUE producer_init_copy(VALUE copy, VALUE orig) {
|
669
|
+
HermannInstanceConfig* orig_config;
|
670
|
+
HermannInstanceConfig* copy_config;
|
671
|
+
|
672
|
+
if(copy == orig) {
|
673
|
+
return copy;
|
674
|
+
}
|
675
|
+
|
676
|
+
if (TYPE(orig) != T_DATA || RDATA(orig)->dfree != (RUBY_DATA_FUNC)producer_free) {
|
677
|
+
rb_raise(rb_eTypeError, "wrong argument type");
|
678
|
+
}
|
679
|
+
|
680
|
+
Data_Get_Struct(orig, HermannInstanceConfig, orig_config);
|
681
|
+
Data_Get_Struct(copy, HermannInstanceConfig, copy_config);
|
682
|
+
|
683
|
+
// Copy over the data from one struct to the other
|
684
|
+
MEMCPY(copy_config, orig_config, HermannInstanceConfig, 1);
|
685
|
+
|
686
|
+
return copy;
|
687
|
+
}
|
688
|
+
|
689
|
+
/**
|
690
|
+
* Init_hermann_lib
|
691
|
+
*
|
692
|
+
* Called by Ruby when the Hermann gem is loaded.
|
693
|
+
* Defines the Hermann module.
|
694
|
+
* Defines the Producer and Consumer classes.
|
695
|
+
*/
|
696
|
+
void Init_hermann_lib() {
|
697
|
+
|
698
|
+
/* Define the module */
|
699
|
+
m_hermann = rb_define_module("Hermann");
|
700
|
+
|
701
|
+
/* ---- Define the consumer class ---- */
|
702
|
+
VALUE c_consumer = rb_define_class_under(m_hermann, "Consumer", rb_cObject);
|
703
|
+
|
704
|
+
/* Allocate */
|
705
|
+
rb_define_alloc_func(c_consumer, consumer_allocate);
|
706
|
+
|
707
|
+
/* Initialize */
|
708
|
+
rb_define_method(c_consumer, "initialize", consumer_initialize, 3);
|
709
|
+
rb_define_method(c_consumer, "initialize_copy", consumer_init_copy, 1);
|
710
|
+
|
711
|
+
/* Consumer has method 'consume' */
|
712
|
+
rb_define_method( c_consumer, "consume", consumer_consume, 0 );
|
713
|
+
|
714
|
+
/* ---- Define the producer class ---- */
|
715
|
+
VALUE c_producer = rb_define_class_under(m_hermann, "Producer", rb_cObject);
|
716
|
+
|
717
|
+
/* Allocate */
|
718
|
+
rb_define_alloc_func(c_producer, producer_allocate);
|
719
|
+
|
720
|
+
/* Initialize */
|
721
|
+
rb_define_method(c_producer, "initialize", producer_initialize, 2);
|
722
|
+
rb_define_method(c_producer, "initialize_copy", producer_init_copy, 1);
|
723
|
+
|
724
|
+
/* Producer.push(msg) */
|
725
|
+
rb_define_method( c_producer, "push", producer_push, 1 );
|
726
|
+
|
727
|
+
}
|
data/ext/hermann_lib.h
ADDED
@@ -0,0 +1,83 @@
|
|
1
|
+
/*
|
2
|
+
* hermann_lib.h - Ruby wrapper for the librdkafka library
|
3
|
+
*
|
4
|
+
* Copyright (c) 2014 Stan Campbell
|
5
|
+
* All rights reserved.
|
6
|
+
*
|
7
|
+
* Redistribution and use in source and binary forms, with or without
|
8
|
+
* modification, are permitted provided that the following conditions are met:
|
9
|
+
*
|
10
|
+
* 1. Redistributions of source code must retain the above copyright notice,
|
11
|
+
* this list of conditions and the following disclaimer.
|
12
|
+
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
13
|
+
* this list of conditions and the following disclaimer in the documentation
|
14
|
+
* and/or other materials provided with the distribution.
|
15
|
+
*
|
16
|
+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
17
|
+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
18
|
+
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
19
|
+
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
20
|
+
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
21
|
+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
22
|
+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
23
|
+
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
24
|
+
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
25
|
+
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
26
|
+
* POSSIBILITY OF SUCH DAMAGE.
|
27
|
+
*/
|
28
|
+
|
29
|
+
#ifndef HERMANN_H
|
30
|
+
#define HERMANN_H
|
31
|
+
|
32
|
+
#include <ruby.h>
|
33
|
+
|
34
|
+
#include <ctype.h>
|
35
|
+
#include <signal.h>
|
36
|
+
#include <string.h>
|
37
|
+
#include <unistd.h>
|
38
|
+
#include <stdlib.h>
|
39
|
+
#include <syslog.h>
|
40
|
+
#include <sys/time.h>
|
41
|
+
#include <errno.h>
|
42
|
+
|
43
|
+
#include <librdkafka/rdkafka.h>
|
44
|
+
|
45
|
+
// Holds the defined Ruby module for Hermann
|
46
|
+
static VALUE m_hermann;
|
47
|
+
|
48
|
+
static int DEBUG = 0;
|
49
|
+
|
50
|
+
// Should we expect rb_thread_blocking_region to be present?
|
51
|
+
// #define RB_THREAD_BLOCKING_REGION
|
52
|
+
#undef RB_THREAD_BLOCKING_REGION
|
53
|
+
|
54
|
+
static enum {
|
55
|
+
OUTPUT_HEXDUMP,
|
56
|
+
OUTPUT_RAW,
|
57
|
+
} output = OUTPUT_HEXDUMP;
|
58
|
+
|
59
|
+
typedef struct HermannInstanceConfig {
|
60
|
+
|
61
|
+
char* topic;
|
62
|
+
|
63
|
+
/* Kafka configuration */
|
64
|
+
rd_kafka_t *rk;
|
65
|
+
rd_kafka_topic_t *rkt;
|
66
|
+
char *brokers;
|
67
|
+
int partition;
|
68
|
+
rd_kafka_topic_conf_t *topic_conf;
|
69
|
+
char errstr[512];
|
70
|
+
rd_kafka_conf_t *conf;
|
71
|
+
const char *debug;
|
72
|
+
int64_t start_offset;
|
73
|
+
int do_conf_dump;
|
74
|
+
|
75
|
+
int run;
|
76
|
+
int exit_eof;
|
77
|
+
int quiet;
|
78
|
+
|
79
|
+
int isInitialized;
|
80
|
+
|
81
|
+
} HermannInstanceConfig;
|
82
|
+
|
83
|
+
#endif
|
data/ext/hermann_lib.o
ADDED
Binary file
|
data/lib/hermann.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require 'hermann_lib'
|
metadata
ADDED
@@ -0,0 +1,74 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: hermann
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 29
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 11
|
9
|
+
version: "0.11"
|
10
|
+
platform: x86-darwin-13
|
11
|
+
authors:
|
12
|
+
- Stan Campbell
|
13
|
+
autorequire:
|
14
|
+
bindir: bin
|
15
|
+
cert_chain: []
|
16
|
+
|
17
|
+
date: 2014-05-29 00:00:00 -07:00
|
18
|
+
default_executable: hermann
|
19
|
+
dependencies: []
|
20
|
+
|
21
|
+
description: Ruby gem wrapper for the RdKafka C library
|
22
|
+
email: stan.campbell3@gmail.com
|
23
|
+
executables: []
|
24
|
+
|
25
|
+
extensions:
|
26
|
+
- ext/extconf.rb
|
27
|
+
extra_rdoc_files: []
|
28
|
+
|
29
|
+
files:
|
30
|
+
- Rakefile
|
31
|
+
- ext/hermann_lib.h
|
32
|
+
- ext/hermann_lib.c
|
33
|
+
- ext/extconf.rb
|
34
|
+
- bin/hermann
|
35
|
+
- lib/hermann.rb
|
36
|
+
- ext/hermann_lib.o
|
37
|
+
- test/test_hermann.rb
|
38
|
+
has_rdoc: true
|
39
|
+
homepage: http://rubygems.org/gems/hermann
|
40
|
+
licenses:
|
41
|
+
- MIT
|
42
|
+
post_install_message:
|
43
|
+
rdoc_options: []
|
44
|
+
|
45
|
+
require_paths:
|
46
|
+
- lib
|
47
|
+
- ext
|
48
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ">="
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
hash: 3
|
54
|
+
segments:
|
55
|
+
- 0
|
56
|
+
version: "0"
|
57
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
58
|
+
none: false
|
59
|
+
requirements:
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
hash: 3
|
63
|
+
segments:
|
64
|
+
- 0
|
65
|
+
version: "0"
|
66
|
+
requirements: []
|
67
|
+
|
68
|
+
rubyforge_project:
|
69
|
+
rubygems_version: 1.6.2
|
70
|
+
signing_key:
|
71
|
+
specification_version: 3
|
72
|
+
summary: The Kafka consumer is based on the librdkafka C library.
|
73
|
+
test_files:
|
74
|
+
- test/test_hermann.rb
|