codeclimate-kafka 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c879ad2675e462df020c9ea423891e439c9a180c
-  data.tar.gz: 17736d6479420bcf9621f328fc8d841386a76074
+  metadata.gz: a041e23f8531766027d023622e51edea283f439e
+  data.tar.gz: 85b0453a007b36211da3d3eb9f324f8a6ac2c623
 SHA512:
-  metadata.gz: 9780a0c3cc0dd70d150fc75b2c9929fad4ff2ad0d9055c9dce5dc38bad6bd2c36083b12253b8019b924ec6374aafa0d7d59c450d0a4fceed36713909259f49d4
-  data.tar.gz: 6ef034f1724bbe081dbcb5cb512d103f550136a7d93b8f84ccaffd436e7940e327d8ce1a5de41ec99d4213f755246acbd6f2ea56d99a206d1ec6665bad6a70ae
+  metadata.gz: fc269f3951c22823b73d8313666de3aebca80f6d54da22319c7891e0cedbe95798f32b1c6132af7ba2365fd46a6526da1544e2a52b99641809eac5c7acd3f734
+  data.tar.gz: 2df20cdc5d45e98a56d588932b14e86527966f85e9536147c09609303d6bc019106292faa044a97608bf2d30057ba7fd2922a24cd60b894733326111f43da6e0

data/lib/cc/kafka.rb

@@ -30,6 +30,10 @@ module CC
       def statsd
         @statsd ||= DummyStatsd.new
       end
+
+      def ssl_certificates
+        @ssl_certificates ||= []
+      end
     end
   end
 end

data/lib/cc/kafka/producer.rb

@@ -1,16 +1,11 @@
 require "bson"
-require "poseidon"
+require "cc/kafka/producer/http"
+require "cc/kafka/producer/poseidon"
 
 module CC
   module Kafka
     class Producer
-      SCHEMES = [
-        HTTP = "http",
-        KAFKA = "kafka",
-      ]
-
-      HTTPError = Class.new(StandardError)
-      HTTP_TIMEOUT= 60 # seconds
+      InvalidScheme = Class.new(StandardError)
 
       def initialize(url, client_id = nil)
         @url = url
@@ -22,65 +17,40 @@ module CC
       def send_message(data, key = nil)
         Kafka.logger.debug("data: #{data.inspect}, key: #{key.inspect}")
 
-        serialized = BSON.serialize(data).to_s
-
-        if http?
-          Kafka.logger.debug("sending message over HTTP")
-          send_http(serialized, key)
-        else
-          Kafka.logger.debug("sending message direct via Poseidon")
-          send_poseidon(serialized, key)
-        end
+        producer.send_message(BSON.serialize(data).to_s, key)
+      rescue
+        producer.close
+        raise
       end
 
       def close
-        unless http?
-          producer.close
-        end
+        producer.close
       end
 
       private
 
-      def http?
-        uri.scheme == HTTP
+      def producer
+        @producer ||= choose_producer
       end
 
-      def send_http(serialized, key)
-        http = Net::HTTP.new(uri.host, uri.port)
-        http.open_timeout = HTTP_TIMEOUT
-        http.read_timeout = HTTP_TIMEOUT
-        request = Net::HTTP::Post.new("/")
-        request["Topic"] = topic
-        request["Key"] = key if key
-        request.body = serialized
-
-        Kafka.logger.debug("POST #{uri.host}:#{uri.port}/")
-        Kafka.logger.debug("data: #{serialized.inspect}")
-        response = http.request(request)
-
-        unless response.is_a?(Net::HTTPSuccess)
-          raise HTTPError, "request not successful: (#{response.code}) #{response.body}"
+      def choose_producer
+        case (scheme = uri.scheme)
+        when "http" then HTTP.new(host, port, topic)
+        when "kafka" then Poseidon.new(host, port, topic, @client_id)
+        else raise InvalidScheme, "invalid scheme #{scheme.inspect}"
         end
       end
 
-      def send_poseidon(serialized, key)
-        message = Poseidon::MessageToSend.new(topic, serialized, key)
-        producer.send_messages([message])
-      rescue
-        producer.close
-        raise
+      def scheme
+        uri.scheme
       end
 
-      def producer
-        @producer ||= Poseidon::Producer.new(
-          broker,
-          @client_id,
-          compression_codec: :gzip
-        )
+      def host
+        uri.host
       end
 
-      def broker
-        ["#{uri.host}:#{uri.port}"]
+      def port
+        uri.port
       end
 
       def topic

data/lib/cc/kafka/producer/http.rb

@@ -0,0 +1,61 @@
+module CC
+  module Kafka
+    class Producer
+      class HTTP
+        HTTPError = Class.new(StandardError)
+        HTTP_TIMEOUT = 60 # seconds
+
+        def initialize(host, port, topic, ssl = false)
+          @host = host
+          @port = port
+          @topic = topic
+          @ssl = ssl
+        end
+
+        def send_message(message, key)
+          Kafka.logger.debug("sending message over HTTP")
+          http = Net::HTTP.new(@host, @port)
+          http.open_timeout = HTTP_TIMEOUT
+          http.read_timeout = HTTP_TIMEOUT
+
+          if ssl?
+            http.use_ssl = true
+            http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+            http.cert_store = certificate_store
+          end
+
+          request = Net::HTTP::Post.new("/")
+          request["Topic"] = @topic
+          request["Key"] = key if key
+          request.body = message
+
+          response = http.request(request)
+
+          unless response.is_a?(Net::HTTPSuccess)
+            raise HTTPError, "request not successful: (#{response.code}) #{response.body}"
+          end
+        end
+
+        def close
+          # no-op
+        end
+
+        private
+
+        def ssl?
+          @ssl
+        end
+
+        def certificate_store
+          OpenSSL::X509::Store.new.tap do |store|
+            store.set_default_paths
+
+            Kafka.ssl_certificates.each do |file|
+              store.add_file(file)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cc/kafka/producer/poseidon.rb

@@ -0,0 +1,36 @@
+require "poseidon"
+
+module CC
+  module Kafka
+    class Producer
+      class Poseidon
+        def initialize(host, port, topic, client_id)
+          @brokers = ["#{host}:#{port}"]
+          @topic = topic
+          @client_id = client_id
+        end
+
+        def send_message(message, key)
+          Kafka.logger.debug("sending message direct via Poseidon")
+          producer.send_messages([
+            ::Poseidon::MessageToSend.new(@topic, message, key)
+          ])
+        end
+
+        def close
+          producer.close
+        end
+
+        private
+
+        def producer
+          @producer ||= ::Poseidon::Producer.new(
+            @brokers,
+            @client_id,
+            compression_codec: :gzip
+          )
+        end
+      end
+    end
+  end
+end

data/lib/cc/kafka/version.rb

@@ -1,5 +1,5 @@
 module CC
   module Kafka
-    VERSION = "0.2.0"
+    VERSION = "0.3.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: codeclimate-kafka
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Code Climate
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-08 00:00:00.000000000 Z
+date: 2015-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bson
@@ -75,6 +75,8 @@ files:
 - lib/cc/kafka.rb
 - lib/cc/kafka/consumer.rb
 - lib/cc/kafka/producer.rb
+- lib/cc/kafka/producer/http.rb
+- lib/cc/kafka/producer/poseidon.rb
 - lib/cc/kafka/version.rb
 homepage: https://codeclimate.com
 licenses: