codeclimate-kafka 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6342040b86ab69ae63452bf69bd757cf1ed9eb84
-  data.tar.gz: 9baba0293728b2935925add35dade45019c0287d
+  metadata.gz: 0b27b99381c9938065fdab7f96159efec38b6e7a
+  data.tar.gz: 88c4167a50b8c5c3cbf3d04fef305392d8d3affb
 SHA512:
-  metadata.gz: bb387152b82a3281fb94020d16904503cb38456bfd409c489c3c6f8ec409b2f8540e77df7b714070040f98c97a47ed15386e085a7ae915390c6254bda128c130
-  data.tar.gz: ebadd12e34eec8f23ca262605dbfaecb87742b18caf35cb46cf599ae59cfd415f46145d13d88c7103aa73c7bb0bcc9b44c1ce39ce3ab3107f0820d442ce61363
+  metadata.gz: 48ebbe244d8ed041b034eca81dc4f4b3fd7c960300fa8a6370167a4075e79fbe1afa6ed919ec21b8011ebb3e0ddebbab07dfd0e60525eff1e491e6f9acf5ff94
+  data.tar.gz: b78c5337244a202ee3c056fd1b727981cc7fc90289d2955315c7ec4e2ea72b888c6da126f340e9c54321b9e1d4d961ac4cf6bbb735a1854be63b3031763fcf68

data/lib/cc/kafka/consumer.rb

@@ -11,14 +11,14 @@ module CC
           partition: partition,
         )
 
-        Kafka.logger.debug("offset: #{@offset.topic}/#{@offset.partition} #{@offset.current}")
+        Kafka.logger.debug("offset: #{@offset.topic}/#{@offset.partition} #{current_offset(@offset)}")
 
         @consumer = Poseidon::PartitionConsumer.consumer_for_partition(
           client_id,
           seed_brokers,
           @offset.topic,
           @offset.partition,
-          @offset.current
+          current_offset(@offset)
         )
       end
 
@@ -46,6 +46,10 @@ module CC
 
       private
 
+      def current_offset(offset)
+        offset.current || :earliest_offset
+      end
+
       def fetch_messages
         @consumer.fetch.each do |message|
           Kafka.statsd.increment("messages.received")

data/lib/cc/kafka/offset_storage/memory.rb

@@ -0,0 +1,19 @@
+module CC
+  module Kafka
+    module OffsetStorage
+      Memory = Struct.new(:topic, :partition, :current) do
+        def self.transaction
+          yield
+        end
+
+        def self.find_or_create!(attrs)
+          @offset ||= new(attrs[:topic], attrs[:partition], nil)
+        end
+
+        def set(attrs)
+          attrs.each { |k, v| self[k] = v }
+        end
+      end
+    end
+  end
+end

data/lib/cc/kafka/offset_storage/minidoc.rb

@@ -0,0 +1,24 @@
+module CC
+  module Kafka
+    module OffsetStorage
+      module Minidoc
+        def self.included(base)
+          base.extend ClassMethods
+          base.include ::Minidoc::Indexes
+
+          base.attribute :topic, String
+          base.attribute :partition, Integer
+          base.attribute :current, Integer
+
+          base.ensure_index [:partition, :topic], unique: true
+        end
+
+        module ClassMethods
+          def find_or_create!(attributes)
+            find_one(attributes) || create!(attributes)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cc/kafka/version.rb

@@ -1,5 +1,5 @@
 module CC
   module Kafka
-    VERSION = "0.4.0"
+    VERSION = "0.5.0"
   end
 end

data/lib/cc/kafka.rb

@@ -1,5 +1,7 @@
 require "logger"
 require "cc/kafka/consumer"
+require "cc/kafka/offset_storage/memory"
+require "cc/kafka/offset_storage/minidoc"
 require "cc/kafka/producer"
 
 module CC

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: codeclimate-kafka
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Code Climate
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-15 00:00:00.000000000 Z
+date: 2015-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bson
@@ -74,6 +74,8 @@ extra_rdoc_files: []
 files:
 - lib/cc/kafka.rb
 - lib/cc/kafka/consumer.rb
+- lib/cc/kafka/offset_storage/memory.rb
+- lib/cc/kafka/offset_storage/minidoc.rb
 - lib/cc/kafka/producer.rb
 - lib/cc/kafka/producer/http.rb
 - lib/cc/kafka/producer/poseidon.rb