ckafka 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c270b9b1b5eec4340d4c34f073650e55855eb54e
-  data.tar.gz: 677a7fca922e3e02fff193c483179b6542bdee15
+  metadata.gz: 211654fb7358899c3c277b80460a270f481c5011
+  data.tar.gz: 22c9d00374d5000d33939fe30c0c13145ab7d73f
 SHA512:
-  metadata.gz: fee2faba877e2153ab619e36ffde79bcf90c05c3c6086923cc7473181fe9c66b77534a9bc121f2aa8fa915a88e1fab6ba34d3accd8dff55423c376d1bdf80ab0
-  data.tar.gz: ff6a04d6f4470e4eeed5195713273090095e18369d0e84abb48fc17cf95ffbeaad95db6f0d69cae0a04a0a26803e2b46be3e9524d014b2b30707f75eeddcf3df
+  metadata.gz: 7eb6d2ea1812e081a818e306926d00413cdc6c6c28f25aebd79a968001b6d3433d7daaec62ef4383108d89e85d55cec4c06fcfbd00ace335ef16108b6819c0df
+  data.tar.gz: 5efd6674116bb4f8fd3e6e88ef44607365d3489b66a56fff6a1819e56039c69ebf4b23562c8dd0bd24ad70a918d369aa1e03984033d8efd6d8f0ef7ab8ad8543

data/.gitignore

@@ -7,4 +7,5 @@
 /pkg/
 /spec/reports/
 /tmp/
+/confluent/
 *.bundle

data/README.md

@@ -1,4 +1,4 @@
-# Ckafka
+# Ckafka ![CircleCI](https://circleci.com/gh/Shopify/ckafka.svg?style=shield)
 
 Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/ckafka`. To experiment with that code, run `bin/console` for an interactive prompt.
 

data/Rakefile

@@ -2,7 +2,7 @@ require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 require 'rake/extensiontask'
 
-RSpec::Core::RakeTask.new(:spec)
+RSpec::Core::RakeTask.new(spec: :compile)
 
 task :default => :spec
 

data/circle.yml

@@ -0,0 +1,27 @@
+machine:
+  ruby:
+    version: 2.0.0
+  environment:
+    LD_LIBRARY_PATH: /usr/local/lib
+
+dependencies:
+  pre:
+    - make confluent/rest/start
+    - confluent/bin/kafka-topics --create --topic test.1  --partitions 1  --replication-factor 1 --zookeeper localhost
+    - >
+      git clone git@github.com:edenhill/librdkafka.git ~/librdkafka;
+      cd ~/librdkafka;
+      git reset --hard 0c15023708302c41a36e95f9650d69c453dfabba;
+      ./configure;
+      make;
+      sudo make install
+  post:
+    - confluent/bin/kafka-topics --list --zookeeper localhost
+
+test:
+  override:
+    - bundle exec rake spec
+
+database:
+  override:
+    - echo "noop"

data/ckafka.gemspec

@@ -19,8 +19,10 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
   spec.extensions = %w(ext/ckafka/extconf.rb)
-  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "bundler", "~> 1.9"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency 'rake-compile'
+  spec.add_development_dependency 'rake-compiler'
+  spec.add_development_dependency 'minitest', '~> 5.8'
+  spec.add_development_dependency "poseidon", "= 0.0.5"
 end

data/ext/ckafka/ckafka.c

@@ -9,28 +9,35 @@
 
 static rd_kafka_t *rk = NULL;
 
+static void error(const char *fmt, ...)
+{
+  va_list args;
+
+  va_start(args, fmt);
+  vfprintf(stderr, fmt, args);
+  va_end(args);
+  fputs("\n", stderr);
+}
+
 static void logger(const rd_kafka_t *rk, int level,
                    const char *fac, const char *buf)
 {
-  fprintf(stderr, "%s: %s\n", fac, buf);
+  fprintf(stderr, "CKafka [%d]: %s: %s\n", level, fac, buf);
 }
 
 
 static VALUE kafka_send(VALUE self, VALUE topic_value, VALUE key, VALUE message)
 {
-  rd_kafka_topic_conf_t *topic_conf;
-  rd_kafka_topic_t *topic;
-  char *topic_name;
-  void *message_bytes;
-  size_t message_len;
-  void *key_buf;
-  size_t key_len;
-  int res;
-
-  if (NIL_P(key)) {
-    key_buf = NULL;
-    key_len = 0;
-  } else {
+  rd_kafka_topic_conf_t *topic_conf = NULL;
+  rd_kafka_topic_t *topic = NULL;
+  char *topic_name = NULL;
+  void *message_bytes = NULL;
+  size_t message_len = 0;
+  void *key_buf = NULL;
+  size_t key_len = 0;
+  int res = 0;
+
+  if (!NIL_P(key)) {
     key_buf = RSTRING_PTR(key);
     key_len = RSTRING_LEN(key);
   }
@@ -40,10 +47,7 @@ static VALUE kafka_send(VALUE self, VALUE topic_value, VALUE key, VALUE message)
     rb_raise(rb_eStandardError, "topic is not a string!");
   }
 
-  if(NIL_P(message)) {
-    message = NULL;
-    message_len = 0;
-  } else {
+  if(!NIL_P(message)) {
     message_bytes = RSTRING_PTR(message);
     if(!message_bytes) {
       rb_raise(rb_eStandardError, "failed to get message ptr");
@@ -65,8 +69,6 @@ static VALUE kafka_send(VALUE self, VALUE topic_value, VALUE key, VALUE message)
   res = rd_kafka_produce(topic, RD_KAFKA_PARTITION_UA, RD_KAFKA_MSG_F_COPY, message_bytes, message_len,
                          key_buf, key_len, NULL);
 
-  rd_kafka_topic_destroy(topic);
-
   if (res) {
     rb_raise(rb_eStandardError, "rd_kafka_produce failed: %d", res);
   }
@@ -77,17 +79,19 @@ static VALUE kafka_send(VALUE self, VALUE topic_value, VALUE key, VALUE message)
 static VALUE kafka_destroy()
 {
   if(rk) {
-    int i;
-
-    for ( i = 0 ; i < MAX_SHUTDOWN_TRIES ; ++i ) {
-      if (rd_kafka_outq_len(rk) <= 0 ) {
+    int i,res;
+    for(i = 0 ; i < MAX_SHUTDOWN_TRIES; ++i) {
+      if(!rd_kafka_outq_len(rk)) {
         break;
-      } else {
-        rd_kafka_poll(rk, 100);
       }
+      rd_kafka_poll(rk, 100);
     }
 
     rd_kafka_destroy(rk);
+    res = rd_kafka_wait_destroyed(100);
+    if(res) {
+      error("wait_destroyed returned: %d\n", res);
+    }
     rk = NULL;
   }
   return Qnil;
@@ -110,27 +114,37 @@ static VALUE kafka_add_broker(VALUE self, VALUE broker)
   return Qnil;
 }
 
-static VALUE kafka_init()
+#define LOG_DEBUG 7
+
+static void error_cb(rd_kafka_t *rk, int err, const char *reason, void *opaque)
+{
+  error("[%d] %s\n", err, reason);
+}
+
+static VALUE kafka_init(VALUE self)
 {
   rd_kafka_conf_t *conf;
   char errstr[512];
 
-  if(!rk) {
+  if(rk) {
     kafka_destroy();
   }
   conf = rd_kafka_conf_new();
 
+  rd_kafka_conf_set_error_cb(conf, error_cb);
   rd_kafka_conf_set_log_cb(conf, logger);
 
   rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr));
   if (!rk) {
     rb_raise(rb_eStandardError, "failed to create kafka producer: %s\n", errstr);
   }
+
   return Qnil;
 }
 
 VALUE Init_ckafka()
 {
+
   VALUE kafka_module = rb_define_module("Ckafka");
 
   rb_define_singleton_method(kafka_module, "init", kafka_init, 0);
@@ -138,7 +152,7 @@ VALUE Init_ckafka()
   rb_define_singleton_method(kafka_module, "add_broker", kafka_add_broker, 1);
   rb_define_singleton_method(kafka_module, "close", kafka_destroy, 0);
 
-  kafka_init();
+  kafka_init(Qnil);
 
   return Qnil;
 }

data/lib/ckafka/version.rb

@@ -1,3 +1,3 @@
 module Ckafka
-  VERSION = "0.2.0"
+  VERSION = '0.3.0'
 end

data/makefile

@@ -0,0 +1,54 @@
+.PHONY: confluent/kafka/* confluent/zookeeper/* confluent/registry/* confluent/start confluent/stop
+
+
+# Confluent platform tasks
+
+confluent/start: confluent/rest/start
+
+confluent/stop: confluent/rest/stop confluent/registry/stop confluent/kafka/stop confluent/zookeeper/stop
+
+# Download & extract tasks
+
+confluent/confluent.tgz:
+	mkdir -p confluent && wget http://packages.confluent.io/archive/1.0/confluent-1.0-2.10.4.tar.gz -O confluent/confluent.tgz
+
+confluent/EXTRACTED: confluent/confluent.tgz
+	tar xzf confluent/confluent.tgz -C confluent --strip-components 1 && mkdir confluent/logs && touch confluent/EXTRACTED
+	echo "delete.topic.enable=true" >> confluent/etc/kafka/server.properties
+	echo "auto.create.topics.enable=false" >> confluent/etc/kafka/server.properties
+
+# Zookeeper tasks
+
+confluent/zookeeper/start: confluent/EXTRACTED
+	nohup confluent/bin/zookeeper-server-start confluent/etc/kafka/zookeeper.properties 2> confluent/logs/zookeeper.err > confluent/logs/zookeeper.out < /dev/null &
+	while ! nc localhost 2181 </dev/null; do echo "Waiting for zookeeper..."; sleep 1; done
+
+confluent/zookeeper/stop: confluent/EXTRACTED
+	confluent/bin/zookeeper-server-stop
+
+# Kafka tasks
+
+confluent/kafka/start: confluent/zookeeper/start confluent/EXTRACTED
+	nohup confluent/bin/kafka-server-start confluent/etc/kafka/server.properties 2> confluent/logs/kafka.err > confluent/logs/kafka.out < /dev/null &
+	while ! nc localhost 9092 </dev/null; do echo "Waiting for Kafka..."; sleep 1; done
+
+confluent/kafka/stop: confluent/EXTRACTED
+	confluent/bin/kafka-server-stop
+
+# schema-registry tasks
+
+confluent/registry/start: confluent/kafka/start confluent/EXTRACTED
+	nohup confluent/bin/schema-registry-start confluent/etc/schema-registry/schema-registry.properties 2> confluent/logs/schema-registry.err > confluent/logs/schema-registry.out < /dev/null &
+	while ! nc localhost 8081 </dev/null; do echo "Waiting for schema registry..."; sleep 1; done
+
+confluent/registry/stop: confluent/EXTRACTED
+	confluent/bin/kafka-server-stop
+
+# REST proxy tasks
+
+confluent/rest/start: confluent/registry/start confluent/EXTRACTED
+	nohup confluent/bin/kafka-rest-start confluent/etc/kafka-rest/kafka-rest.properties 2> confluent/logs/kafka-rest.err > confluent/logs/kafka-rest.out < /dev/null &
+	while ! nc localhost 8082 </dev/null; do echo "Waiting for REST proxy..."; sleep 1; done
+
+confluent/rest/stop: confluent/EXTRACTED
+	confluent/bin/kafka-rest-stop

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ckafka
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Ian Quick
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-05-05 00:00:00.000000000 Z
+date: 2016-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '1.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '1.9'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -53,7 +53,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: rake-compile
+  name: rake-compiler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -66,6 +66,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.8'
+- !ruby/object:Gem::Dependency
+  name: poseidon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.5
 description: interface to the rdkafka C library
 email:
 - ian.quick@gmail.com
@@ -83,11 +111,13 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- circle.yml
 - ckafka.gemspec
 - ext/ckafka/ckafka.c
 - ext/ckafka/extconf.rb
 - lib/ckafka.rb
 - lib/ckafka/version.rb
+- makefile
 homepage: https://github.com/ibawt/ckafka
 licenses:
 - MIT