hermann 0.11-x86-darwin-13

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,31 @@
1
+
2
+ require 'rake/clean'
3
+
4
+ EXT_CONF = "ext/extconf.rb"
5
+ MAKEFILE = 'ext/Makefile'
6
+ MODULE = 'ext/hermann_lib.so'
7
+ SRC = Dir.glob('ext/*.c')
8
+ SRC << MAKEFILE
9
+
10
+ CLEAN.include [ 'ext/*.o', 'ext/depend', 'ext/hermann_lib.bundle', MODULE ]
11
+ CLOBBER.include [ 'config.save', 'ext/mkmf.log', 'ext/hermann_lib.bundle', MAKEFILE ]
12
+
13
+ file MAKEFILE => EXT_CONF do |t|
14
+ Dir::chdir(File::dirname(EXT_CONF)) do
15
+ unless sh "ruby #{File::basename(EXT_CONF)}"
16
+ $stderr.puts "Failed to run extconf"
17
+ break
18
+ end
19
+ end
20
+ end
21
+
22
+ file MODULE => SRC do |t|
23
+ Dir::chdir(File::dirname(EXT_CONF)) do
24
+ unless sh "make"
25
+ $stderr.puts "make failed"
26
+ end
27
+ end
28
+ end
29
+
30
+ desc "Build the native library"
31
+ task :build => MODULE
@@ -0,0 +1,4 @@
1
+ #!ruby
2
+
3
+ require 'hermann'
4
+ puts "Placeholder..."
@@ -0,0 +1,32 @@
1
+ # External configuration for Hermann Gem
2
+
3
+ require 'mkmf'
4
+
5
+ RbConfig::MAKEFILE_CONFIG['CC'] = ENV['CC'] if ENV['CC']
6
+
7
+ LIBDIR = RbConfig::CONFIG['libdir']
8
+ INCLUDEDIR = RbConfig::CONFIG['includedir']
9
+
10
+ puts "Library Dir: #{LIBDIR}\n"
11
+ puts "Include Dir: #{INCLUDEDIR}"
12
+
13
+ HEADER_DIRS = [INCLUDEDIR]
14
+
15
+ LIB_DIRS = [LIBDIR]
16
+
17
+ dir_config('rdkafka', HEADER_DIRS, LIB_DIRS)
18
+
19
+ unless find_header('librdkafka/rdkafka.h')
20
+ abort "librdkafka not installed"
21
+ end
22
+
23
+ unless find_library('rdkafka', 'rd_kafka_conf_new')
24
+ abort "librdkafka not installed"
25
+ end
26
+
27
+ #unless have_func "rb_thread_blocking_region"
28
+ # abort "rb_thread_blocking_region function missing"
29
+ #end
30
+
31
+ # create_header('hermann_lib.h')
32
+ create_makefile('hermann/hermann_lib')
@@ -0,0 +1,727 @@
1
+ /*
2
+ * hermann_lib.c - Ruby wrapper for the librdkafka library
3
+ *
4
+ * Copyright (c) 2014 Stan Campbell
5
+ * All rights reserved.
6
+ *
7
+ * Redistribution and use in source and binary forms, with or without
8
+ * modification, are permitted provided that the following conditions are met:
9
+ *
10
+ * 1. Redistributions of source code must retain the above copyright notice,
11
+ * this list of conditions and the following disclaimer.
12
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
13
+ * this list of conditions and the following disclaimer in the documentation
14
+ * and/or other materials provided with the distribution.
15
+ *
16
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26
+ * POSSIBILITY OF SUCH DAMAGE.
27
+ */
28
+
29
+ /* Much of the librdkafka library calls were lifted from rdkafka_example.c */
30
+
31
+ #include "hermann_lib.h"
32
+
33
+ /**
34
+ * Utility functions
35
+ */
36
+
37
+
38
+ /**
39
+ * Convenience function
40
+ *
41
+ * @param msg char* the string to be logged under debugging.
42
+ */
43
+ void log_debug(char* msg) {
44
+ if(DEBUG) {
45
+ fprintf(stderr, "%s\n", msg);
46
+ }
47
+ }
48
+
49
+ /**
50
+ * Message delivery report callback.
51
+ * Called once for each message.
52
+ *
53
+ * @param rk rd_kafka_t* instance of producer or consumer
54
+ * @param payload void* the payload of the message
55
+ * @param len size_t the length of the payload in bytes
56
+ * @param error_code int
57
+ * @param opaque void* optional context
58
+ * @param msg_opaque void* it's opaque
59
+ */
60
+ static void msg_delivered (rd_kafka_t *rk,
61
+ void *payload, size_t len,
62
+ int error_code,
63
+ void *opaque, void *msg_opaque) {
64
+
65
+ if (error_code)
66
+ fprintf(stderr, "%% Message delivery failed: %s\n",
67
+ rd_kafka_err2str(error_code));
68
+ }
69
+
70
+ /**
71
+ * Producer partitioner callback.
72
+ * Used to determine the target partition within a topic for production.
73
+ *
74
+ * Returns an integer partition number or RD_KAFKA_PARTITION_UA if no
75
+ * available partition could be determined.
76
+ *
77
+ * @param rkt rd_kafka_topic_t* the topic
78
+ * @param keydata void* key information for calculating the partition
79
+ * @param keylen size_t key size
80
+ * @param partition_cnt int32_t the count of the number of partitions
81
+ * @param rkt_opaque void* opaque topic info
82
+ * @param msg_opaque void* opaque message info
83
+ */
84
+ static int32_t producer_paritioner_callback( const rd_kafka_topic_t *rkt,
85
+ const void *keydata,
86
+ size_t keylen,
87
+ int32_t partition_cnt,
88
+ void *rkt_opaque,
89
+ void *msg_opaque) {
90
+ /* Pick a random partition */
91
+ int retry;
92
+ for(retry=0;retry<partition_cnt;retry++) {
93
+ int32_t partition = rand() % partition_cnt;
94
+ if(rd_kafka_topic_partition_available(rkt, partition)) {
95
+ break; /* this one will do */
96
+ }
97
+ }
98
+ }
99
+
100
+ /**
101
+ * hexdump
102
+ *
103
+ * Write the given payload to file in hex notation.
104
+ *
105
+ * @param fp FILE* the file into which to write
106
+ * @param name char* name
107
+ * @param ptr void* payload
108
+ * @param len size_t payload length
109
+ */
110
+ static void hexdump (FILE *fp, const char *name, const void *ptr, size_t len) {
111
+ const char *p = (const char *)ptr;
112
+ int of = 0;
113
+
114
+
115
+ if (name)
116
+ fprintf(fp, "%s hexdump (%zd bytes):\n", name, len);
117
+
118
+ for (of = 0 ; of < len ; of += 16) {
119
+ char hexen[16*3+1];
120
+ char charen[16+1];
121
+ int hof = 0;
122
+
123
+ int cof = 0;
124
+ int i;
125
+
126
+ for (i = of ; i < of + 16 && i < len ; i++) {
127
+ hof += sprintf(hexen+hof, "%02x ", p[i] & 0xff);
128
+ cof += sprintf(charen+cof, "%c",
129
+ isprint((int)p[i]) ? p[i] : '.');
130
+ }
131
+ fprintf(fp, "%08x: %-48s %-16s\n",
132
+ of, hexen, charen);
133
+ }
134
+ }
135
+
136
+ /**
137
+ * msg_consume
138
+ *
139
+ * Callback on message receipt.
140
+ *
141
+ * @param rkmessage rd_kafka_message_t* the message
142
+ * @param opaque void* opaque context
143
+ */
144
+ static void msg_consume (rd_kafka_message_t *rkmessage,
145
+ void *opaque) {
146
+
147
+ HermannInstanceConfig* cfg;
148
+
149
+ cfg = (HermannInstanceConfig*)opaque;
150
+
151
+ if (rkmessage->err) {
152
+ if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
153
+ fprintf(stderr,
154
+ "%% Consumer reached end of %s [%"PRId32"] "
155
+ "message queue at offset %"PRId64"\n",
156
+ rd_kafka_topic_name(rkmessage->rkt),
157
+ rkmessage->partition, rkmessage->offset);
158
+
159
+ if (cfg->exit_eof)
160
+ cfg->run = 0;
161
+
162
+ return;
163
+ }
164
+
165
+ fprintf(stderr, "%% Consume error for topic \"%s\" [%"PRId32"] "
166
+ "offset %"PRId64": %s\n",
167
+ rd_kafka_topic_name(rkmessage->rkt),
168
+ rkmessage->partition,
169
+ rkmessage->offset,
170
+ rd_kafka_message_errstr(rkmessage));
171
+ return;
172
+ }
173
+
174
+ if (DEBUG && rkmessage->key_len) {
175
+ if (output == OUTPUT_HEXDUMP)
176
+ hexdump(stdout, "Message Key",
177
+ rkmessage->key, rkmessage->key_len);
178
+ else
179
+ printf("Key: %.*s\n",
180
+ (int)rkmessage->key_len, (char *)rkmessage->key);
181
+ }
182
+
183
+ if (output == OUTPUT_HEXDUMP) {
184
+ if(DEBUG)
185
+ hexdump(stdout, "Message Payload", rkmessage->payload, rkmessage->len);
186
+ } else {
187
+ if(DEBUG)
188
+ printf("%.*s\n", (int)rkmessage->len, (char *)rkmessage->payload);
189
+ }
190
+
191
+ // Yield the data to the Consumer's block
192
+ if(rb_block_given_p()) {
193
+ VALUE value = rb_str_new((char *)rkmessage->payload, rkmessage->len);
194
+ rb_yield(value);
195
+ } else {
196
+ if(DEBUG)
197
+ fprintf(stderr, "No block given\n"); // todo: should this be an error?
198
+ }
199
+ }
200
+
201
+ /**
202
+ * logger
203
+ *
204
+ * Kafka logger callback (optional)
205
+ *
206
+ * todo: introduce better logging
207
+ *
208
+ * @param rk rd_kafka_t the producer or consumer
209
+ * @param level int the log level
210
+ * @param fac char* something of which I am unaware
211
+ * @param buf char* the log message
212
+ */
213
+ static void logger (const rd_kafka_t *rk, int level,
214
+ const char *fac, const char *buf) {
215
+ struct timeval tv;
216
+ gettimeofday(&tv, NULL);
217
+ fprintf(stderr, "%u.%03u RDKAFKA-%i-%s: %s: %s\n",
218
+ (int)tv.tv_sec, (int)(tv.tv_usec / 1000),
219
+ level, fac, rd_kafka_name(rk), buf);
220
+ }
221
+
222
+ /**
223
+ * consumer_init_kafka
224
+ *
225
+ * Initialize the Kafka context and instantiate a consumer.
226
+ *
227
+ * @param config HermannInstanceConfig* pointer to the instance configuration for this producer or consumer
228
+ */
229
+ void consumer_init_kafka(HermannInstanceConfig* config) {
230
+
231
+ config->quiet = !isatty(STDIN_FILENO);
232
+
233
+ /* Kafka configuration */
234
+ config->conf = rd_kafka_conf_new();
235
+
236
+ /* Topic configuration */
237
+ config->topic_conf = rd_kafka_topic_conf_new();
238
+
239
+ /* Create Kafka handle */
240
+ if (!(config->rk = rd_kafka_new(RD_KAFKA_CONSUMER, config->conf,
241
+ config->errstr, sizeof(config->errstr)))) {
242
+ fprintf(stderr, "%% Failed to create new consumer: %s\n", config->errstr);
243
+ exit(1);
244
+ }
245
+
246
+ /* Set logger */
247
+ rd_kafka_set_logger(config->rk, logger);
248
+ rd_kafka_set_log_level(config->rk, LOG_DEBUG);
249
+
250
+ /* TODO: offset calculation */
251
+ config->start_offset = RD_KAFKA_OFFSET_END;
252
+
253
+ /* Add brokers */
254
+ if(rd_kafka_brokers_add(config->rk, config->brokers) == 0) {
255
+ fprintf(stderr, "%% No valid brokers specified\n");
256
+ exit(1);
257
+ }
258
+
259
+ /* Create topic */
260
+ config->rkt = rd_kafka_topic_new(config->rk, config->topic, config->topic_conf);
261
+
262
+ /* We're now initialized */
263
+ config->isInitialized = 1;
264
+ }
265
+
266
+ // Ruby gem extensions
267
+
268
+ /**
269
+ * Callback invoked if Ruby needs to stop our Consumer's IO loop for any reason (system exit, etc.)
270
+ */
271
+ static void consumer_consume_stop_callback(void *ptr) {
272
+ HermannInstanceConfig* config = (HermannInstanceConfig*)ptr;
273
+
274
+ config->run = 0;
275
+ }
276
+
277
+ /**
278
+ * Loop on a timeout to receive messages from Kafka. When the consumer_consume_stop_callback is invoked by Ruby,
279
+ * we'll break out of our loop and return.
280
+ */
281
+ void consumer_consume_loop(HermannInstanceConfig* consumerConfig) {
282
+
283
+ while (consumerConfig->run) {
284
+ rd_kafka_message_t *rkmessage;
285
+
286
+ if(rd_kafka_consume_callback(consumerConfig->rkt, consumerConfig->partition,
287
+ 1000/*timeout*/,
288
+ msg_consume,
289
+ consumerConfig) < 0) {
290
+ fprintf(stderr, "%% Error: %s\n", rd_kafka_err2str( rd_kafka_errno2err(errno)));
291
+ }
292
+
293
+ }
294
+ }
295
+
296
+ /**
297
+ * Hermann::Consumer.consume
298
+ *
299
+ * Begin listening on the configured topic for messages. msg_consume will be called on each message received.
300
+ *
301
+ * @param VALUE self the Ruby object for this consumer
302
+ */
303
+ static VALUE consumer_consume(VALUE self) {
304
+
305
+ HermannInstanceConfig* consumerConfig;
306
+
307
+ Data_Get_Struct(self, HermannInstanceConfig, consumerConfig);
308
+
309
+ if(consumerConfig->topic==NULL) {
310
+ fprintf(stderr, "Topic is null!");
311
+ return;
312
+ }
313
+
314
+ if(!consumerConfig->isInitialized) {
315
+ consumer_init_kafka(consumerConfig);
316
+ }
317
+
318
+ /* Start consuming */
319
+ if (rd_kafka_consume_start(consumerConfig->rkt, consumerConfig->partition, consumerConfig->start_offset) == -1){
320
+ fprintf(stderr, "%% Failed to start consuming: %s\n",
321
+ rd_kafka_err2str(rd_kafka_errno2err(errno)));
322
+ exit(1);
323
+ }
324
+
325
+ #ifdef RB_THREAD_BLOCKING_REGION
326
+ /** The consumer will listen for incoming messages in a loop, timing out and checking the consumerConfig->run
327
+ * flag every second.
328
+ *
329
+ * Call rb_thread_blocking_region to release the GVM lock and allow Ruby to amuse itself while we wait on
330
+ * IO from Kafka.
331
+ *
332
+ * If Ruby needs to interrupt the consumer loop, the stop callback will be invoked and the loop should exit.
333
+ */
334
+ rb_thread_blocking_region(consumer_consume_loop, consumerConfig, consumer_consume_stop_callback, consumerConfig);
335
+ #else
336
+ consumer_consume_loop(consumerConfig);
337
+ #endif
338
+
339
+
340
+ /* Stop consuming */
341
+ rd_kafka_consume_stop(consumerConfig->rkt, consumerConfig->partition);
342
+
343
+ return Qnil;
344
+ }
345
+
346
+ /**
347
+ * producer_init_kafka
348
+ *
349
+ * Initialize the producer instance, setting up the Kafka topic and context.
350
+ *
351
+ * @param config HermannInstanceConfig* the instance configuration associated with this producer.
352
+ */
353
+ void producer_init_kafka(HermannInstanceConfig* config) {
354
+
355
+ config->quiet = !isatty(STDIN_FILENO);
356
+
357
+ /* Kafka configuration */
358
+ config->conf = rd_kafka_conf_new();
359
+
360
+ /* Topic configuration */
361
+ config->topic_conf = rd_kafka_topic_conf_new();
362
+
363
+ /* Set up a message delivery report callback.
364
+ * It will be called once for each message, either on successful
365
+ * delivery to broker, or upon failure to deliver to broker. */
366
+ rd_kafka_conf_set_dr_cb(config->conf, msg_delivered);
367
+
368
+ /* Create Kafka handle */
369
+ if (!(config->rk = rd_kafka_new(RD_KAFKA_PRODUCER, config->conf, config->errstr, sizeof(config->errstr)))) {
370
+ fprintf(stderr,
371
+ "%% Failed to create new producer: %s\n", config->errstr);
372
+ exit(1);
373
+ }
374
+
375
+ /* Set logger */
376
+ rd_kafka_set_logger(config->rk, logger);
377
+ rd_kafka_set_log_level(config->rk, LOG_DEBUG);
378
+
379
+ if(rd_kafka_brokers_add(config->rk, config->brokers) == 0) {
380
+ fprintf(stderr, "%% No valid brokers specified\n");
381
+ exit(1);
382
+ }
383
+
384
+ /* Create topic */
385
+ config->rkt = rd_kafka_topic_new(config->rk, config->topic, config->topic_conf);
386
+
387
+ /* Set the partitioner callback */
388
+ rd_kafka_topic_conf_set_partitioner_cb( config->topic_conf, producer_paritioner_callback );
389
+
390
+ /* We're now initialized */
391
+ config->isInitialized = 1;
392
+ }
393
+
394
+ /**
395
+ * producer_push_single
396
+ *
397
+ * @param self VALUE the Ruby producer instance
398
+ * @param message VALUE the ruby String containing the outgoing message.
399
+ */
400
+ static VALUE producer_push_single(VALUE self, VALUE message) {
401
+
402
+ HermannInstanceConfig* producerConfig;
403
+ char buf[2048];
404
+
405
+ Data_Get_Struct(self, HermannInstanceConfig, producerConfig);
406
+
407
+ if(producerConfig->topic==NULL) {
408
+ fprintf(stderr, "Topic is null!");
409
+ return self;
410
+ }
411
+
412
+ if(!producerConfig->isInitialized) {
413
+ producer_init_kafka(producerConfig);
414
+ }
415
+
416
+ char *msg = StringValueCStr(message);
417
+ strcpy(buf, msg);
418
+
419
+ size_t len = strlen(buf);
420
+ if (buf[len-1] == '\n')
421
+ buf[--len] = '\0';
422
+
423
+ /* Send/Produce message. */
424
+ if (rd_kafka_produce(producerConfig->rkt, producerConfig->partition, RD_KAFKA_MSG_F_COPY,
425
+ /* Payload and length */
426
+ buf, len,
427
+ /* Optional key and its length */
428
+ NULL, 0,
429
+ /* Message opaque, provided in
430
+ * delivery report callback as
431
+ * msg_opaque. */
432
+ NULL) == -1) {
433
+
434
+ fprintf(stderr, "%% Failed to produce to topic %s partition %i: %s\n",
435
+ rd_kafka_topic_name(producerConfig->rkt), producerConfig->partition,
436
+ rd_kafka_err2str(rd_kafka_errno2err(errno)));
437
+
438
+ /* Poll to handle delivery reports */
439
+ rd_kafka_poll(producerConfig->rk, 10);
440
+ }
441
+
442
+ /* Must poll to handle delivery reports */
443
+ rd_kafka_poll(producerConfig->rk, 0);
444
+
445
+ return self;
446
+ }
447
+
448
+ /**
449
+ * producer_push_array
450
+ *
451
+ * Publish each of the messages in array on the configured topic.
452
+ *
453
+ * @param self VALUE the instance of the Ruby Producer object
454
+ * @param length int the length of the outgoing messages array
455
+ * @param array VALUE the Ruby array of messages
456
+ */
457
+ static VALUE producer_push_array(VALUE self, int length, VALUE array) {
458
+
459
+ int i;
460
+ VALUE message;
461
+
462
+ for(i=0;i<length;i++) {
463
+ message = RARRAY_PTR(array)[i];
464
+ producer_push_single(self, message);
465
+ }
466
+
467
+ return self;
468
+ }
469
+
470
+ /**
471
+ * Hermann::Producer.push(msg)
472
+ *
473
+ * Publish the given message on the configured topic.
474
+ *
475
+ * @param self VALUE the Ruby instance of the Producer.
476
+ * @param message VALUE the Ruby string containing the message.
477
+ */
478
+ static VALUE producer_push(VALUE self, VALUE message) {
479
+
480
+ VALUE arrayP = rb_check_array_type(message);
481
+
482
+ if(!NIL_P(arrayP)) {
483
+ return producer_push_array(self, RARRAY_LEN(arrayP), message);
484
+ } else {
485
+ return producer_push_single(self, message);
486
+ }
487
+ }
488
+
489
+ /**
490
+ * consumer_free
491
+ *
492
+ * Callback called when Ruby needs to GC the configuration associated with an Hermann instance.
493
+ *
494
+ * @param p void* the instance of an HermannInstanceConfig to be freed from allocated memory.
495
+ */
496
+ static void consumer_free(void * p) {
497
+
498
+ HermannInstanceConfig* config = (HermannInstanceConfig *)p;
499
+
500
+ // the p *should* contain a pointer to the consumerConfig which also must be freed
501
+ rd_kafka_topic_destroy(config->rkt);
502
+
503
+ rd_kafka_destroy(config->rk);
504
+
505
+ // clean up the struct
506
+ free(config);
507
+ }
508
+
509
+ /**
510
+ * consumer_allocate
511
+ *
512
+ * Allocate and wrap an HermannInstanceConfig for this Consumer object.
513
+ *
514
+ * @param klass VALUE the class of the enclosing Ruby object.
515
+ */
516
+ static VALUE consumer_allocate(VALUE klass) {
517
+
518
+ VALUE obj;
519
+
520
+ HermannInstanceConfig* consumerConfig = ALLOC(HermannInstanceConfig);
521
+ obj = Data_Wrap_Struct(klass, 0, consumer_free, consumerConfig);
522
+
523
+ return obj;
524
+ }
525
+
526
+ /**
527
+ * consumer_initialize
528
+ *
529
+ * todo: configure the brokers through passed parameter, later through zk
530
+ *
531
+ * Set up the Consumer's HermannInstanceConfig context.
532
+ *
533
+ * @param self VALUE the Ruby instance of the Consumer
534
+ * @param topic VALUE a Ruby string
535
+ * @param brokers VALUE a Ruby string containing list of host:port
536
+ * @param partition VALUE a Ruby number
537
+ */
538
+ static VALUE consumer_initialize(VALUE self, VALUE topic, VALUE brokers, VALUE partition) {
539
+
540
+ HermannInstanceConfig* consumerConfig;
541
+ char* topicPtr;
542
+ char* brokersPtr;
543
+ int partitionNo;
544
+
545
+ topicPtr = StringValuePtr(topic);
546
+ brokersPtr = StringValuePtr(brokers);
547
+ partitionNo = FIX2INT(partition);
548
+ Data_Get_Struct(self, HermannInstanceConfig, consumerConfig);
549
+
550
+ consumerConfig->topic = topicPtr;
551
+ consumerConfig->brokers = brokersPtr;
552
+ consumerConfig->partition = partitionNo;
553
+ consumerConfig->run = 1;
554
+ consumerConfig->exit_eof = 0;
555
+ consumerConfig->quiet = 0;
556
+
557
+ return self;
558
+ }
559
+
560
+ /**
561
+ * consumer_init_copy
562
+ *
563
+ * When copying into a new instance of a Consumer, reproduce the configuration info.
564
+ *
565
+ * @param copy VALUE the Ruby Consumer instance (with configuration) as destination
566
+ * @param orig VALUE the Ruby Consumer instance (with configuration) as source
567
+ *
568
+ */
569
+ static VALUE consumer_init_copy(VALUE copy, VALUE orig) {
570
+ HermannInstanceConfig* orig_config;
571
+ HermannInstanceConfig* copy_config;
572
+
573
+ if(copy == orig) {
574
+ return copy;
575
+ }
576
+
577
+ if (TYPE(orig) != T_DATA || RDATA(orig)->dfree != (RUBY_DATA_FUNC)consumer_free) {
578
+ rb_raise(rb_eTypeError, "wrong argument type");
579
+ }
580
+
581
+ Data_Get_Struct(orig, HermannInstanceConfig, orig_config);
582
+ Data_Get_Struct(copy, HermannInstanceConfig, copy_config);
583
+
584
+ // Copy over the data from one struct to the other
585
+ MEMCPY(copy_config, orig_config, HermannInstanceConfig, 1);
586
+
587
+ return copy;
588
+ }
589
+
590
+ /**
591
+ * producer_free
592
+ *
593
+ * Reclaim memory allocated to the Producer's configuration
594
+ *
595
+ * @param p void* the instance's configuration struct
596
+ */
597
+ static void producer_free(void * p) {
598
+
599
+ HermannInstanceConfig* config = (HermannInstanceConfig *)p;
600
+
601
+ // Clean up the topic
602
+ rd_kafka_topic_destroy(config->rkt);
603
+
604
+ // Take care of the producer instance
605
+ rd_kafka_destroy(config->rk);
606
+
607
+ // Free the struct
608
+ free(config);
609
+ }
610
+
611
+ /**
612
+ * producer_allocate
613
+ *
614
+ * Allocate the memory for a Producer's configuration
615
+ *
616
+ * @param klass VALUE the class of the Producer
617
+ */
618
+ static VALUE producer_allocate(VALUE klass) {
619
+
620
+ VALUE obj;
621
+
622
+ HermannInstanceConfig* producerConfig = ALLOC(HermannInstanceConfig);
623
+ obj = Data_Wrap_Struct(klass, 0, producer_free, producerConfig);
624
+
625
+ return obj;
626
+ }
627
+
628
+ /**
629
+ * producer_initialize
630
+ *
631
+ * Set up the configuration context for the Producer instance
632
+ *
633
+ * @param self VALUE the Producer instance
634
+ * @param topic VALUE the Ruby string naming the topic
635
+ * @param brokers VALUE a Ruby string containing host:port pairs separated by commas
636
+ */
637
+ static VALUE producer_initialize(VALUE self, VALUE topic, VALUE brokers) {
638
+
639
+ HermannInstanceConfig* producerConfig;
640
+ char* topicPtr;
641
+ char* brokersPtr;
642
+
643
+ topicPtr = StringValuePtr(topic);
644
+ brokersPtr = StringValuePtr(brokers);
645
+ Data_Get_Struct(self, HermannInstanceConfig, producerConfig);
646
+
647
+ producerConfig->topic = topicPtr;
648
+ producerConfig->brokers = brokersPtr;
649
+ /** Using RD_KAFKA_PARTITION_UA specifies we want the partitioner callback to be called to determine the target
650
+ * partition
651
+ */
652
+ producerConfig->partition = RD_KAFKA_PARTITION_UA;
653
+ producerConfig->run = 1;
654
+ producerConfig->exit_eof = 0;
655
+ producerConfig->quiet = 0;
656
+
657
+ return self;
658
+ }
659
+
660
+ /**
661
+ * producer_init_copy
662
+ *
663
+ * Copy the configuration information from orig into copy for the given Producer instances.
664
+ *
665
+ * @param copy VALUE destination Producer
666
+ * @param orign VALUE source Producer
667
+ */
668
+ static VALUE producer_init_copy(VALUE copy, VALUE orig) {
669
+ HermannInstanceConfig* orig_config;
670
+ HermannInstanceConfig* copy_config;
671
+
672
+ if(copy == orig) {
673
+ return copy;
674
+ }
675
+
676
+ if (TYPE(orig) != T_DATA || RDATA(orig)->dfree != (RUBY_DATA_FUNC)producer_free) {
677
+ rb_raise(rb_eTypeError, "wrong argument type");
678
+ }
679
+
680
+ Data_Get_Struct(orig, HermannInstanceConfig, orig_config);
681
+ Data_Get_Struct(copy, HermannInstanceConfig, copy_config);
682
+
683
+ // Copy over the data from one struct to the other
684
+ MEMCPY(copy_config, orig_config, HermannInstanceConfig, 1);
685
+
686
+ return copy;
687
+ }
688
+
689
+ /**
690
+ * Init_hermann_lib
691
+ *
692
+ * Called by Ruby when the Hermann gem is loaded.
693
+ * Defines the Hermann module.
694
+ * Defines the Producer and Consumer classes.
695
+ */
696
+ void Init_hermann_lib() {
697
+
698
+ /* Define the module */
699
+ m_hermann = rb_define_module("Hermann");
700
+
701
+ /* ---- Define the consumer class ---- */
702
+ VALUE c_consumer = rb_define_class_under(m_hermann, "Consumer", rb_cObject);
703
+
704
+ /* Allocate */
705
+ rb_define_alloc_func(c_consumer, consumer_allocate);
706
+
707
+ /* Initialize */
708
+ rb_define_method(c_consumer, "initialize", consumer_initialize, 3);
709
+ rb_define_method(c_consumer, "initialize_copy", consumer_init_copy, 1);
710
+
711
+ /* Consumer has method 'consume' */
712
+ rb_define_method( c_consumer, "consume", consumer_consume, 0 );
713
+
714
+ /* ---- Define the producer class ---- */
715
+ VALUE c_producer = rb_define_class_under(m_hermann, "Producer", rb_cObject);
716
+
717
+ /* Allocate */
718
+ rb_define_alloc_func(c_producer, producer_allocate);
719
+
720
+ /* Initialize */
721
+ rb_define_method(c_producer, "initialize", producer_initialize, 2);
722
+ rb_define_method(c_producer, "initialize_copy", producer_init_copy, 1);
723
+
724
+ /* Producer.push(msg) */
725
+ rb_define_method( c_producer, "push", producer_push, 1 );
726
+
727
+ }
@@ -0,0 +1,83 @@
1
+ /*
2
+ * hermann_lib.h - Ruby wrapper for the librdkafka library
3
+ *
4
+ * Copyright (c) 2014 Stan Campbell
5
+ * All rights reserved.
6
+ *
7
+ * Redistribution and use in source and binary forms, with or without
8
+ * modification, are permitted provided that the following conditions are met:
9
+ *
10
+ * 1. Redistributions of source code must retain the above copyright notice,
11
+ * this list of conditions and the following disclaimer.
12
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
13
+ * this list of conditions and the following disclaimer in the documentation
14
+ * and/or other materials provided with the distribution.
15
+ *
16
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26
+ * POSSIBILITY OF SUCH DAMAGE.
27
+ */
28
+
29
+ #ifndef HERMANN_H
30
+ #define HERMANN_H
31
+
32
+ #include <ruby.h>
33
+
34
+ #include <ctype.h>
35
+ #include <signal.h>
36
+ #include <string.h>
37
+ #include <unistd.h>
38
+ #include <stdlib.h>
39
+ #include <syslog.h>
40
+ #include <sys/time.h>
41
+ #include <errno.h>
42
+
43
+ #include <librdkafka/rdkafka.h>
44
+
45
+ // Holds the defined Ruby module for Hermann
46
+ static VALUE m_hermann;
47
+
48
+ static int DEBUG = 0;
49
+
50
+ // Should we expect rb_thread_blocking_region to be present?
51
+ // #define RB_THREAD_BLOCKING_REGION
52
+ #undef RB_THREAD_BLOCKING_REGION
53
+
54
+ static enum {
55
+ OUTPUT_HEXDUMP,
56
+ OUTPUT_RAW,
57
+ } output = OUTPUT_HEXDUMP;
58
+
59
+ typedef struct HermannInstanceConfig {
60
+
61
+ char* topic;
62
+
63
+ /* Kafka configuration */
64
+ rd_kafka_t *rk;
65
+ rd_kafka_topic_t *rkt;
66
+ char *brokers;
67
+ int partition;
68
+ rd_kafka_topic_conf_t *topic_conf;
69
+ char errstr[512];
70
+ rd_kafka_conf_t *conf;
71
+ const char *debug;
72
+ int64_t start_offset;
73
+ int do_conf_dump;
74
+
75
+ int run;
76
+ int exit_eof;
77
+ int quiet;
78
+
79
+ int isInitialized;
80
+
81
+ } HermannInstanceConfig;
82
+
83
+ #endif
Binary file
@@ -0,0 +1 @@
1
+ require 'hermann_lib'
@@ -0,0 +1,9 @@
1
+ require 'test/unit'
2
+ require 'hermann'
3
+
4
+ class HermannTest < Test::Unit::TestCase
5
+
6
+ def test_stub
7
+ assert_equal "Apples", "Apples"
8
+ end
9
+ end
metadata ADDED
@@ -0,0 +1,74 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: hermann
3
+ version: !ruby/object:Gem::Version
4
+ hash: 29
5
+ prerelease:
6
+ segments:
7
+ - 0
8
+ - 11
9
+ version: "0.11"
10
+ platform: x86-darwin-13
11
+ authors:
12
+ - Stan Campbell
13
+ autorequire:
14
+ bindir: bin
15
+ cert_chain: []
16
+
17
+ date: 2014-05-29 00:00:00 -07:00
18
+ default_executable: hermann
19
+ dependencies: []
20
+
21
+ description: Ruby gem wrapper for the RdKafka C library
22
+ email: stan.campbell3@gmail.com
23
+ executables: []
24
+
25
+ extensions:
26
+ - ext/extconf.rb
27
+ extra_rdoc_files: []
28
+
29
+ files:
30
+ - Rakefile
31
+ - ext/hermann_lib.h
32
+ - ext/hermann_lib.c
33
+ - ext/extconf.rb
34
+ - bin/hermann
35
+ - lib/hermann.rb
36
+ - ext/hermann_lib.o
37
+ - test/test_hermann.rb
38
+ has_rdoc: true
39
+ homepage: http://rubygems.org/gems/hermann
40
+ licenses:
41
+ - MIT
42
+ post_install_message:
43
+ rdoc_options: []
44
+
45
+ require_paths:
46
+ - lib
47
+ - ext
48
+ required_ruby_version: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ">="
52
+ - !ruby/object:Gem::Version
53
+ hash: 3
54
+ segments:
55
+ - 0
56
+ version: "0"
57
+ required_rubygems_version: !ruby/object:Gem::Requirement
58
+ none: false
59
+ requirements:
60
+ - - ">="
61
+ - !ruby/object:Gem::Version
62
+ hash: 3
63
+ segments:
64
+ - 0
65
+ version: "0"
66
+ requirements: []
67
+
68
+ rubyforge_project:
69
+ rubygems_version: 1.6.2
70
+ signing_key:
71
+ specification_version: 3
72
+ summary: The Kafka consumer is based on the librdkafka C library.
73
+ test_files:
74
+ - test/test_hermann.rb