codeclimate-kafka 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 819817bf067f7aa8de2c99aeeed13a72e83f76c6
+  data.tar.gz: d24a7b691c331a90b77747162d6753d3b0679709
+SHA512:
+  metadata.gz: 1bbbf6cc5e3a7abdbeaab47bc2431569b5b5a184ef9e0e02eb10258e2aae3259c44b0f944a86b39c1f018a43ef935edfff1c99f20923edf51e5a20ca84e2fd20
+  data.tar.gz: 26fe00dcd28113be6568552bb623d2839c3fc7c8de1a082682d01cc45cc9409b17fceb8e40c3db6c41c4edfd99b1fee270a55eb5a4b7af8977ab544e1505da5d

data/lib/cc/kafka.rb

@@ -0,0 +1,35 @@
+require "logger"
+require "cc/kafka/consumer"
+require "cc/kafka/producer"
+
+module CC
+  module Kafka
+    ConfigurationError = Class.new(StandardError)
+
+    class DummyStatsd
+      def method_missing(*)
+        yield if block_given?
+      end
+    end
+
+    class << self
+      attr_writer :offset_model, :logger, :statsd
+
+      def logger
+        @logger ||= Logger.new(STDOUT)
+      end
+
+      def offset_model
+        if @offset_model.nil?
+          raise ConfigurationError, "Kafka.offset_model not set"
+        end
+
+        @offset_model
+      end
+
+      def statsd
+        @statsd ||= DummyStatsd.new
+      end
+    end
+  end
+end

data/lib/cc/kafka/consumer.rb

@@ -0,0 +1,67 @@
+require "poseidon"
+
+module CC
+  module Kafka
+    class Consumer
+      def initialize(client_id, seed_brokers, topic, partition)
+        @offset = Kafka.offset_model.find_or_create!(
+          topic: topic,
+          partition: partition,
+        )
+
+        Kafka.logger.debug("offset: #{@offset.topic}/#{@offset.partition} #{@offset.current}")
+
+        @consumer = Poseidon::PartitionConsumer.consumer_for_partition(
+          client_id,
+          seed_brokers,
+          @offset.topic,
+          @offset.partition,
+          @offset.current
+        )
+      end
+
+      def on_message(&block)
+        @on_message = block
+      end
+
+      def start
+        trap(:TERM) { stop }
+
+        @running = true
+
+        while @running do
+          fetch_messages
+        end
+
+        Kafka.logger.info("shutting down due to TERM signal")
+
+      ensure
+        @consumer.close
+      end
+
+      def stop
+        @running = false
+      end
+
+      private
+
+      def fetch_messages
+        @consumer.fetch.each do |message|
+          Kafka.statsd.increment("messages.received")
+          Kafka.statsd.time("messages.processing") do
+            @offset.set(current: message.offset + 1)
+
+            Kafka.offset_model.transaction do
+              @on_message.call(BSON.deserialize(message.value))
+            end
+          end
+          Kafka.statsd.increment("messages.processed")
+        end
+      rescue Poseidon::Errors::UnknownTopicOrPartition
+        Kafka.logger.debug("topic #{@topic} not created yet")
+
+        sleep 1
+      end
+    end
+  end
+end

data/lib/cc/kafka/producer.rb

@@ -0,0 +1,50 @@
+require "bson"
+require "poseidon"
+
+module CC
+  module Kafka
+    class Producer
+      def initialize(client_id, url)
+        @client_id = client_id
+        @url = url
+      end
+
+      def send_message(data, key = nil)
+        serialized = BSON.serialize(data).to_s
+        message = Poseidon::MessageToSend.new(topic, serialized, key)
+
+        producer.send_messages([message])
+      rescue
+        close
+
+        raise
+      end
+
+      def close
+        producer.close
+      end
+
+      private
+
+      def producer
+        @producer ||= Poseidon::Producer.new(
+          broker,
+          @client_id,
+          compression_codec: :gzip
+        )
+      end
+
+      def broker
+        ["#{uri.host}:#{uri.port}"]
+      end
+
+      def topic
+        uri.path.split("/")[1]
+      end
+
+      def uri
+        @uri ||= URI.parse(@url)
+      end
+    end
+  end
+end

data/lib/cc/kafka/version.rb

@@ -0,0 +1,5 @@
+module CC
+  module Kafka
+    VERSION = "0.0.1"
+  end
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification
+name: codeclimate-kafka
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Code Climate
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-06-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bson
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.12.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.12.2
+- !ruby/object:Gem::Dependency
+  name: bson_ext
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.12.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.12.2
+- !ruby/object:Gem::Dependency
+  name: poseidon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Code Climate Kafka Client
+email: hello@codeclimate.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/cc/kafka.rb
+- lib/cc/kafka/consumer.rb
+- lib/cc/kafka/producer.rb
+- lib/cc/kafka/version.rb
+homepage: https://codeclimate.com
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Code Climate Kafka Client
+test_files: []