ruby-kafka 0.5.2.beta3 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d68ed75fbf6a3af99f593f269f46c28bb1353468
-  data.tar.gz: 1f7f32e97728d5f262262329952ef763e01d6c34
+  metadata.gz: 045d5a61276b8499bdb55c05602625a44f558b38
+  data.tar.gz: 5192d0caa0e53cb995ff315aedbb35de61b214f0
 SHA512:
-  metadata.gz: 5d697badab0b48c5fc7c0abc33ea9ed55ea3532309aae0f8ce620ab3dab89ac14f7fc3d020cf299575afe9c38f9f7d46f343ad9de1d809dcbddc7ec513fbc19b
-  data.tar.gz: 481f5ea4a504b96b7610eebca186a3eb791f48655bfe88e268bd01f070dfec3243f4a515a4dd37d79e782f29d7219618d3d5a925c37505102fc3e75cc6fe426c
+  metadata.gz: bfa82cc9074c2e8c3e8e82cd1dfe288fae912fc02a96d355ebcf69d0106746749c00b5120b6a7a151b5c8191ed499d52c1ee0543675772f7792d38d83d2b0ced
+  data.tar.gz: b03a4e3b7b50c6a0ec879c23a29059ced2f242ae8c5e6e6239afc9e9e083fcc0960b07b03b7ed9cd47c7a3f3c7855fc856448974b9ef5757f6500d741821edba

data/.readygo

@@ -0,0 +1 @@
+{"readygo_file_format_version":1,"benchmark_results":[["message serialization decoding",[0.00777734375,0.007609375,0.00721875,0.00836328125,0.007546875,0.0088046875,0.0069414062500000005,0.007550781249999999,0.0072734374999999995,0.009625,0.007867187500000001,0.01226953125,0.0073984375,0.007292968750000001,0.00937890625,0.0073750000000000005]]]}

data/CHANGELOG.md

@@ -4,6 +4,8 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+## v0.5.2
+
 - Instrument the start of message/batch processing (#496).
 - Mark `Client#fetch_messages` as stable.
 - Fix the list topics API (#508).
@@ -12,6 +14,7 @@ Changes and additions to the library will be listed here.
 - Fix compressed message set offset bug (#506).
 - Test against multiple versions of Kafka.
 - Fix double-processing of messages after a consumer exception (#518).
+- Track consumer offsets in Datadog.
 
 ## v0.5.1
 

data/benchmarks/message_encoding.rb

@@ -0,0 +1,21 @@
+require "kafka"
+
+ready "message serialization" do
+  before do
+    message = Kafka::Protocol::Message.new(
+      value: "hello",
+      key: "world",
+    )
+
+    @io = StringIO.new
+    encoder = Kafka::Protocol::Encoder.new(@io)
+    message.encode(encoder)
+
+    @decoder = Kafka::Protocol::Decoder.new(@io)
+  end
+
+  go "decoding" do
+    @io.rewind
+    Kafka::Protocol::Message.decode(@decoder)
+  end
+end

data/lib/kafka/consumer.rb

@@ -294,6 +294,7 @@ module Kafka
             notification = {
               topic: batch.topic,
               partition: batch.partition,
+              last_offset: batch.last_offset,
               offset_lag: batch.offset_lag,
               highwater_mark_offset: batch.highwater_mark_offset,
               message_count: batch.messages.count,

data/lib/kafka/datadog.rb

@@ -122,6 +122,7 @@ module Kafka
 
     class ConsumerSubscriber < StatsdSubscriber
       def process_message(event)
+        offset = event.payload.fetch(:offset)
         offset_lag = event.payload.fetch(:offset_lag)
         create_time = event.payload.fetch(:create_time)
         time_lag = create_time && ((Time.now - create_time) * 1000).to_i
@@ -140,6 +141,7 @@ module Kafka
           increment("consumer.messages", tags: tags)
         end
 
+        gauge("consumer.offset", offset, tags: tags)
         gauge("consumer.lag", offset_lag, tags: tags)
 
         # Not all messages have timestamps.
@@ -149,6 +151,7 @@ module Kafka
       end
 
       def process_batch(event)
+        offset = event.payload.fetch(:last_offset)
         lag = event.payload.fetch(:offset_lag)
         messages = event.payload.fetch(:message_count)
 
@@ -166,6 +169,7 @@ module Kafka
           count("consumer.messages", messages, tags: tags)
         end
 
+        gauge("consumer.offset", offset, tags: tags)
         gauge("consumer.lag", lag, tags: tags)
       end
 

data/lib/kafka/fetch_operation.rb

@@ -91,12 +91,9 @@ module Kafka
 
             messages = fetched_partition.messages.map {|message|
               FetchedMessage.new(
-                value: message.value,
-                key: message.key,
+                message: message,
                 topic: fetched_topic.name,
                 partition: fetched_partition.partition,
-                offset: message.offset,
-                create_time: message.create_time,
               )
             }
 

data/lib/kafka/fetched_message.rb

@@ -1,31 +1,35 @@
 module Kafka
   class FetchedMessage
-
-    # @return [String] the value of the message.
-    attr_reader :value
-
-    # @return [String] the key of the message.
-    attr_reader :key
-
     # @return [String] the name of the topic that the message was written to.
     attr_reader :topic
 
     # @return [Integer] the partition number that the message was written to.
     attr_reader :partition
 
+    def initialize(message:, topic:, partition:)
+      @message = message
+      @topic = topic
+      @partition = partition
+    end
+
+    # @return [String] the value of the message.
+    def value
+      @message.value
+    end
+
+    # @return [String] the key of the message.
+    def key
+      @message.key
+    end
+
     # @return [Integer] the offset of the message in the partition.
-    attr_reader :offset
+    def offset
+      @message.offset
+    end
 
     # @return [Time] the timestamp of the message.
-    attr_reader :create_time
-
-    def initialize(value: nil, key: nil, topic:, partition:, offset:, create_time: nil)
-      @value = value
-      @key = key
-      @topic = topic
-      @partition = partition
-      @offset = offset
-      @create_time = create_time
+    def create_time
+      @message.create_time
     end
   end
 end

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.5.2.beta3"
+  VERSION = "0.5.2"
 end

data/ruby-kafka.gemspec

@@ -43,5 +43,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "ruby-prof"
   spec.add_development_dependency "timecop"
   spec.add_development_dependency "rubocop", "~> 0.49.1"
-  spec.add_development_dependency "gssapi", '>=1.2.0'
+  spec.add_development_dependency "gssapi", ">= 1.2.0"
+  spec.add_development_dependency "stackprof"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.5.2.beta3
+  version: 0.5.2
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-01-04 00:00:00.000000000 Z
+date: 2018-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -262,6 +262,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.0
+- !ruby/object:Gem::Dependency
+  name: stackprof
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A client library for the Kafka distributed commit log.
 email:
 - daniel.schierbeck@gmail.com
@@ -271,6 +285,7 @@ extra_rdoc_files: []
 files:
 - ".circleci/config.yml"
 - ".gitignore"
+- ".readygo"
 - ".rspec"
 - ".rubocop.yml"
 - ".yardopts"
@@ -281,6 +296,7 @@ files:
 - Procfile
 - README.md
 - Rakefile
+- benchmarks/message_encoding.rb
 - bin/console
 - bin/setup
 - ci/consumer.rb
@@ -369,19 +385,6 @@ files:
 - lib/ruby-kafka.rb
 - performance/profile.rb
 - ruby-kafka.gemspec
-- vendor/bundle/bin/bundler
-- vendor/bundle/bin/coderay
-- vendor/bundle/bin/dotenv
-- vendor/bundle/bin/htmldiff
-- vendor/bundle/bin/ldiff
-- vendor/bundle/bin/pry
-- vendor/bundle/bin/rake
-- vendor/bundle/bin/rspec
-- vendor/bundle/bin/rubocop
-- vendor/bundle/bin/ruby-parse
-- vendor/bundle/bin/ruby-prof
-- vendor/bundle/bin/ruby-prof-check-trace
-- vendor/bundle/bin/ruby-rewrite
 homepage: https://github.com/zendesk/ruby-kafka
 licenses:
 - Apache License Version 2.0
@@ -397,9 +400,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.13

data/vendor/bundle/bin/bundler

@@ -1,17 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-#
-# This file was generated by Bundler.
-#
-# The application 'bundler' is installed as part of a gem, and
-# this file is here to facilitate running it.
-#
-
-require "pathname"
-ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../../Gemfile",
-  Pathname.new(__FILE__).realpath)
-
-require "rubygems"
-require "bundler/setup"
-
-load Gem.bin_path("bundler", "bundler")

data/vendor/bundle/bin/coderay

@@ -1,17 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-#
-# This file was generated by Bundler.
-#
-# The application 'coderay' is installed as part of a gem, and
-# this file is here to facilitate running it.
-#
-
-require "pathname"
-ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../../Gemfile",
-  Pathname.new(__FILE__).realpath)
-
-require "rubygems"
-require "bundler/setup"
-
-load Gem.bin_path("coderay", "coderay")

data/vendor/bundle/bin/dotenv

@@ -1,17 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-#
-# This file was generated by Bundler.
-#
-# The application 'dotenv' is installed as part of a gem, and
-# this file is here to facilitate running it.
-#
-
-require "pathname"
-ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../../Gemfile",
-  Pathname.new(__FILE__).realpath)
-
-require "rubygems"
-require "bundler/setup"
-
-load Gem.bin_path("dotenv", "dotenv")

data/vendor/bundle/bin/htmldiff

@@ -1,17 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-#
-# This file was generated by Bundler.
-#
-# The application 'htmldiff' is installed as part of a gem, and
-# this file is here to facilitate running it.
-#
-
-require "pathname"
-ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../../Gemfile",
-  Pathname.new(__FILE__).realpath)
-
-require "rubygems"
-require "bundler/setup"
-
-load Gem.bin_path("diff-lcs", "htmldiff")

data/vendor/bundle/bin/ldiff

@@ -1,17 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-#
-# This file was generated by Bundler.
-#
-# The application 'ldiff' is installed as part of a gem, and
-# this file is here to facilitate running it.
-#
-
-require "pathname"
-ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../../Gemfile",
-  Pathname.new(__FILE__).realpath)
-
-require "rubygems"
-require "bundler/setup"
-
-load Gem.bin_path("diff-lcs", "ldiff")

data/vendor/bundle/bin/pry

@@ -1,17 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-#
-# This file was generated by Bundler.
-#
-# The application 'pry' is installed as part of a gem, and
-# this file is here to facilitate running it.
-#
-
-require "pathname"
-ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../../Gemfile",
-  Pathname.new(__FILE__).realpath)
-
-require "rubygems"
-require "bundler/setup"
-
-load Gem.bin_path("pry", "pry")

data/vendor/bundle/bin/rake

@@ -1,17 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-#
-# This file was generated by Bundler.
-#
-# The application 'rake' is installed as part of a gem, and
-# this file is here to facilitate running it.
-#
-
-require "pathname"
-ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../../Gemfile",
-  Pathname.new(__FILE__).realpath)
-
-require "rubygems"
-require "bundler/setup"
-
-load Gem.bin_path("rake", "rake")

data/vendor/bundle/bin/rspec

@@ -1,17 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-#
-# This file was generated by Bundler.
-#
-# The application 'rspec' is installed as part of a gem, and
-# this file is here to facilitate running it.
-#
-
-require "pathname"
-ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../../Gemfile",
-  Pathname.new(__FILE__).realpath)
-
-require "rubygems"
-require "bundler/setup"
-
-load Gem.bin_path("rspec-core", "rspec")

data/vendor/bundle/bin/rubocop

@@ -1,17 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-#
-# This file was generated by Bundler.
-#
-# The application 'rubocop' is installed as part of a gem, and
-# this file is here to facilitate running it.
-#
-
-require "pathname"
-ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../../Gemfile",
-  Pathname.new(__FILE__).realpath)
-
-require "rubygems"
-require "bundler/setup"
-
-load Gem.bin_path("rubocop", "rubocop")

data/vendor/bundle/bin/ruby-parse

@@ -1,17 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-#
-# This file was generated by Bundler.
-#
-# The application 'ruby-parse' is installed as part of a gem, and
-# this file is here to facilitate running it.
-#
-
-require "pathname"
-ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../../Gemfile",
-  Pathname.new(__FILE__).realpath)
-
-require "rubygems"
-require "bundler/setup"
-
-load Gem.bin_path("parser", "ruby-parse")

data/vendor/bundle/bin/ruby-prof

@@ -1,17 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-#
-# This file was generated by Bundler.
-#
-# The application 'ruby-prof' is installed as part of a gem, and
-# this file is here to facilitate running it.
-#
-
-require "pathname"
-ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../../Gemfile",
-  Pathname.new(__FILE__).realpath)
-
-require "rubygems"
-require "bundler/setup"
-
-load Gem.bin_path("ruby-prof", "ruby-prof")

data/vendor/bundle/bin/ruby-prof-check-trace

@@ -1,17 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-#
-# This file was generated by Bundler.
-#
-# The application 'ruby-prof-check-trace' is installed as part of a gem, and
-# this file is here to facilitate running it.
-#
-
-require "pathname"
-ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../../Gemfile",
-  Pathname.new(__FILE__).realpath)
-
-require "rubygems"
-require "bundler/setup"
-
-load Gem.bin_path("ruby-prof", "ruby-prof-check-trace")

data/vendor/bundle/bin/ruby-rewrite

@@ -1,17 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-#
-# This file was generated by Bundler.
-#
-# The application 'ruby-rewrite' is installed as part of a gem, and
-# this file is here to facilitate running it.
-#
-
-require "pathname"
-ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../../Gemfile",
-  Pathname.new(__FILE__).realpath)
-
-require "rubygems"
-require "bundler/setup"
-
-load Gem.bin_path("parser", "ruby-rewrite")