kafka 0.5.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b1ff8fcc9cb3c43bd818733a5af934490298e0fc28a54668bb4a47d24abe8c69
+  data.tar.gz: 9ca67cd457ceadaffaea6bdb2b1c61d1f4481aa3e2a0ce3874bfb251c3ae9e1f
+SHA512:
+  metadata.gz: 0ecdfd53f2621b6f876595a159847683a8360d2cfa675b0161db5baf688415f2d65c195848d0835c8dbd2aad06ed3377d54dde463bad3e83968893778ef143c6
+  data.tar.gz: c56a0ceafce7c89761aad6da5bdd45ca0da82a765d7072ea472abd6c8fc585dceebaf8f37f2783ea8cccb18205f58f0ef883f16472a8693860a59bcb1697b49a

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/Gemfile.lock
+
+# Ignore librdkafka build artifacts
+/ext/librdkafka.*
+/ext/tmp
+/ext/ports

data/.rubocop.yml

@@ -0,0 +1,210 @@
+AllCops:
+  TargetRubyVersion: 2.6
+  Exclude:
+    - "bin/*"
+    - "ext/tmp/*"
+    - "ext/ports/*"
+
+require:
+  - rubocop-performance
+
+#################
+# [i] Overrides #
+#################
+
+Layout/ArgumentAlignment:
+  EnforcedStyle: with_fixed_indentation
+
+Layout/HashAlignment:
+  EnforcedLastArgumentHashStyle: ignore_implicit
+
+# Align ends correctly.
+Layout/EndAlignment:
+  EnforcedStyleAlignWith: variable
+
+Layout/LineLength:
+  Max: 200
+
+Layout/SpaceInsideBlockBraces:
+  SpaceBeforeBlockParameters: true
+
+Layout/SpaceInsideHashLiteralBraces:
+  EnforcedStyle: space
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+
+Style/SymbolArray:
+  EnforcedStyle: brackets
+
+Style/CollectionMethods:
+  # Mapping from undesired method to desired_method
+  # e.g. to use `detect` over `find`:
+  #
+  # CollectionMethods:
+  #   PreferredMethods:
+  #     find: detect
+  PreferredMethods:
+    reduce: 'inject'
+    find: 'detect'
+    each_with_index: 'each.with_index'
+
+Style/StringLiteralsInInterpolation:
+  EnforcedStyle: double_quotes
+
+Style/SignalException:
+  EnforcedStyle: only_raise
+
+# Require that rescue blocks use the inferred StandardError rather than require
+# the class to be specified.
+Style/RescueStandardError:
+  EnforcedStyle: "implicit"
+
+Style/TrailingCommaInArrayLiteral:
+  Enabled: true
+  EnforcedStyleForMultiline: consistent_comma
+
+Style/TrailingCommaInHashLiteral:
+  Enabled: true
+  EnforcedStyleForMultiline: consistent_comma
+
+Style/TrailingCommaInArguments:
+  EnforcedStyleForMultiline: comma
+
+Layout/FirstHashElementIndentation:
+  EnforcedStyle: consistent
+
+Layout/FirstArrayElementIndentation:
+  EnforcedStyle: consistent
+
+Layout/MultilineMethodCallIndentation:
+  EnforcedStyle: indented
+
+Style/PercentLiteralDelimiters:
+  PreferredDelimiters:
+    "%w": "[]"
+    "%W": "[]"
+    "%i": "[]"
+
+Style/EmptyElse:
+  EnforcedStyle: empty
+
+Style/EmptyMethod:
+  EnforcedStyle: expanded
+
+Metrics/PerceivedComplexity:
+  Max: 11
+
+# As a wrapper around librdkafka there is little we can do to properly limit
+# this.
+Metrics/ParameterLists:
+  Max: 7
+
+#################
+# Disabled cops #
+#################
+Style/ClassAndModuleChildren:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/CyclomaticComplexity:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/EachWithObject:
+  Enabled: false
+
+Style/FormatString:
+  Enabled: false
+
+Style/Lambda:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Style/NegatedIf:
+  Enabled: false
+
+Style/Semicolon:
+  Enabled: false
+
+Layout/SpaceInsideArrayLiteralBrackets:
+  Enabled: false
+
+Style/SingleLineBlockParams:
+  Enabled: false
+
+Style/WordArray:
+  Enabled: false
+
+Style/IfUnlessModifier:
+  Enabled: false
+
+Style/NumericLiterals:
+  Enabled: false
+
+Style/TrailingUnderscoreVariable:
+  Enabled: false
+
+Style/EmptyCaseCondition:
+  Enabled: false
+
+Style/WhileUntilModifier:
+  Enabled: false
+
+Lint/AssignmentInCondition:
+  Enabled: false
+
+# It was recommending parenthasizing blocks that were not ambiguous.
+Lint/AmbiguousBlockAssociation:
+  Enabled: false
+
+Metrics/AbcSize:
+  Enabled: false
+
+Style/AsciiComments:
+  Enabled: false
+
+Performance/TimesMap:
+  Enabled: false
+
+Style/GuardClause:
+  Enabled: false
+
+Style/BracesAroundHashParameters:
+  Enabled: false
+
+Style/NumericPredicate:
+  Enabled: false
+
+Style/SafeNavigation:
+  Enabled: false
+
+# Fails for a pretty consistent pattern where we preemptively use a return in a
+# conditional to allow future expansion of a method:
+#
+# def foo
+#   if condition?
+#     render action: :bar
+#     return
+#    end
+#  end
+Style/RedundantReturn:
+  Enabled: false
+
+# The goal of this cop is to improve readability for formatting strings and
+# could be useful for complex formats. As of 2019-12-28 all of the offending
+# instances are simple single value formats and wouldn't be any more readable.
+Style/FormatStringToken:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,45 @@
+dist: bionic
+
+language: ruby
+rvm:
+  - 2.5
+  - 2.6
+  - 2.7
+  - ruby-head
+
+# Use a consistent Kafka version for each version of Ruby being tested.
+env:
+  - KAFKA=2.2
+
+# Run tests against different versions of Kafka using whatever RVM considers
+# the most recent stable version of Ruby to be.
+jobs:
+  allow_failures:
+    - rvm: ruby-head
+  include:
+    - rvm: ruby
+      env: KAFKA=2.1
+    - rvm: ruby
+      pnv: KAFKA=2.2
+    - rvm: ruby
+      env: KAFKA=2.3
+    - rvm: ruby
+      env: KAFKA=2.4
+
+services:
+  - docker
+
+cache: bundler
+
+before_install:
+  - docker-compose -f spec/support/kafka-$KAFKA.yml up -d
+  - gem install bundler -v 2.1.3
+
+  # Install an up to date version of kafkacat via docker. The version available
+  # in apt doesn't support nil keys or headers.
+  - docker pull edenhill/kafkacat:1.5.0
+  - echo 'docker run -i --rm --network=host edenhill/kafkacat:1.5.0 "$@"' > $HOME/bin/kafkacat
+  - chmod +x $HOME/bin/kafkacat
+
+script:
+  - bundle exec rake

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.5.0 / 2020-01-21
+
+* Initial release!

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at gaffneyc@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [https://contributor-covenant.org/version/1/4][version]
+
+[homepage]: https://contributor-covenant.org
+[version]: https://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Chris Gaffney
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,182 @@
+# Kafka
+
+[![Build Status](https://travis-ci.com/deadmanssnitch/kafka.svg?branch=master)](https://travis-ci.com/deadmanssnitch/kafka)
+
+The kafka gem provides a general producer and consumer for
+[Apache Kafka](https://kafka.apache.org) using bindings to the official
+[C client librdkafka](https://github.com/edenhill/librdkafka).  The `Kafka::FFI`
+module implements an object oriented mapping to most of the librdkafka API,
+making it easier and safer to use than calling functions directly.
+
+## ⚠️ Project Status: Beta ⚠️
+
+This project is currently of BETA quality. Some APIs are still in flux and may
+change. There are also probably a number of subtle (and not so subtle) bugs and
+memory leaks.  Since this relies heavily on binding to librdkafka through FFI
+there are probably code paths which will cause segfaults or memory corruption.
+
+Working with Kafka::FFI directly has many sharp edges which are blunted by
+everything in the Kafka module.
+
+You (yes you!) can make a difference and help make this project better. Test
+against your application and traffic, implement missing functions (see
+`rake ffi:missing`), work with the API and make suggestions for improvements.
+All help is wanted and appreciated.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem "kafka"
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install kafka
+
+## Usage
+
+For more examples see [the examples directory](examples/).
+
+For a detailed introduction on librdkafka which would be useful when working
+with `Kafka::FFI` directly, see 
+[the librdkafka documentation](https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md).
+
+### Sending Message to a Topic
+
+```ruby
+require "kafka"
+
+config = Kafka::Config.new("bootstrap.servers": "localhost:9092")
+producer = Kafka::Producer.new(config)
+
+# Asynchronously publish a JSON payload to the events topic.
+event = { time: Time.now, status: "success" }
+result = producer.produce("events", event.to_json)
+
+# Wait for the delivery to confirm that publishing was successful.
+# result.wait
+# result.successful?
+```
+
+### Consuming Messages from a Topic
+
+```ruby
+require "kafka"
+
+config = Kafka::Config.new({
+  "bootstrap.servers": "localhost:9092",
+
+  # Required for consumers to know what consumer group to join.
+  "group.id": "web.production.eventer",
+})
+
+consumer = Kafka::Consumer.new(config)
+consumer.subscribe("events")
+
+@run = true
+trap("INT")  { @run = false }
+trap("TERM") { @run = false }
+
+while @run
+  consumer.poll do |message|
+    puts message.payload
+  end
+end
+```
+
+### Configuration
+
+Kafka has a lot of potential knobs to turn and dials to tweak. A
+`Kafka::Config` uses the same configuration options as librdkafka (and most or
+all from the Java client). The defaults are generally good and a fine place to
+start.
+
+[All Configuration Options](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md)
+
+### Bindings
+
+`Kafka::FFI` provides bindings to functions in
+[librdkafka](https://github.com/edenhill/librdkafka/blob/master/src/rdkafka.h).
+All of the names are the same and mostly have named parameters to help with
+calling them. Be aware that you will need to handle some memory management to
+call most functions exported in the bindings. See
+[rdkafka.h](https://github.com/edenhill/librdkafka/blob/master/src/rdkafka.h)
+for any questions about usage and semantics.
+
+All classes in `Kafka::FFI` provide an object oriented mapping to the functions
+exported on `Kafka::FFI.rd_kafka_*`. Most will require understanding memory
+management but most should be easier to use and safe than calling into
+librdkafka directly.
+
+## Why another Kafka gem?
+
+There are already at least two good gems for Kafka:
+[ruby-kafka](https://github.com/zendesk/ruby-kafka) and
+[rdkafka](https://github.com/appsignal/rdkafka-ruby). In fact we've used both
+of these gems on Dead Man's Snitch for quite a while and they've been great. We
+really appreciate all of the work that has gone into them :heart:.
+
+Unfortunately, keeping up with Kafka feature and protocol changes can be a full
+time job. Development on ruby-kafka has stalled for that reason and many
+consumer/producer libraries are migrating away from it.
+
+As a heartbeat and cron job monitoring service, we depend on receiving and
+processing reports from jobs reliably and quickly. Failing to receive a report
+could mean waking someone up at 3AM or forcing them to take time away from
+family or friends to deal with a false alarm. What started as a deep dive into
+rdkafka to understand how best to use it reliably, we had ideas we wanted to
+implement that probably wouldn't have been a good fit for rdkafka so we decided
+to start from scratch.
+
+Our goal is to provide a stable and easy to maintain Kafka consumer / producer
+for Ruby. With time as our biggest constraint it makes sense to leverage
+librdkafka as it has full time maintenance and support by the team behind
+Kafka. FFI makes it fast and easy to expose new librdkafka APIs as they are
+added. A stable test suite means being able to meaningfully spend the limited
+amount of time we have available to invest. Embracing memory management and
+building clean separations between layers should reduce the burden to implement
+new bindings as the rules and responsibilities of each layer are clear.
+
+## Development
+
+To get started with development make sure to have docker, docker-compose, and
+[kafkacat](https://github.com/edenhill/kafkacat) installed as they make getting
+up to speed easier.
+
+Before running the test, start a Kafka broker instance
+
+```console
+rake kafka:up
+```
+
+Then run the tests with
+```console
+rake
+```
+
+When you're done shut down the Kafka instance by running:
+```console
+rake kafka:down
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at
+https://github.com/deadmanssnitch/kafka. This project is intended to be a safe,
+welcoming space for collaboration, and contributors are expected to adhere to
+the [code of conduct](https://github.com/deadmanssnitch/kafka/blob/master/CODE_OF_CONDUCT.md).
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the Kafka project's codebases and issue trackers are expected to follow the
+[code of conduct](https://github.com/deadmanssnitch/kafka/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+desc "Build librdkafka into ext"
+task :ext do
+  if Dir.glob("ext/librdkafka.*").empty?
+    sh "cd ext && rake"
+  end
+end
+
+task default: [:ext, :spec]
+
+namespace :ffi do
+  desc "Lists the librdkafka functions that have not been implemented in Kafka::FFI"
+  task :missing do
+    require_relative "lib/kafka/version"
+
+    require "uri"
+    require "net/http"
+    require "tempfile"
+
+    header = Tempfile.new(["rdkafka", ".h"])
+
+    # Fetch the header for the pinned version of librdkafka. rdkafka.h contains
+    # all of the exported function prototypes.
+    url = URI("https://raw.githubusercontent.com/edenhill/librdkafka/v#{::Kafka::LIBRDKAFKA_VERSION}/src/rdkafka.h")
+    resp = Net::HTTP.get(url)
+    header.write(resp)
+    header.close
+
+    all = `ctags -x --sort=yes --kinds-C=pf #{header.path} | awk '{ print $1 }'`
+    all = all.split("\n")
+
+    ffi_path = File.expand_path("lib/kafka/ffi.rb", __dir__)
+    implemented = `grep -o -h -P '^\\s+attach_function\\s+:\\Krd_kafka_\\w+' #{ffi_path}`
+    implemented = implemented.split("\n").sort
+
+    missing = all - implemented
+    puts missing
+  ensure
+    header.unlink
+  end
+
+  desc "Prints the list of implemented librdkafka functions"
+  task :implemented do
+    ffi_path = File.expand_path("lib/kafka/ffi.rb", __dir__)
+    puts `grep -o -h -P '^\\s+attach_function\\s+:\\Krd_kafka_\\w+' #{ffi_path} | sort`
+  end
+end
+
+namespace :kafka do
+  desc "Start an instance of Kafka running in docker"
+  task :up do
+    # Find the docker-compose file for the most recent version of Kafka in
+    # spec/support.
+    compose = Dir["spec/support/kafka-*.yml"].max
+
+    sh "docker-compose -p ruby_kafka_dev -f #{compose} up -d"
+  end
+
+  desc "Shutdown the development Kafka instance"
+  task :down do
+    sh "docker-compose -p ruby_kafka_dev down"
+  end
+end

data/examples/consumer.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.expand_path("../lib", __dir__)
+
+require "kafka"
+
+config = Kafka::Config.new({
+  "bootstrap.servers": "127.0.0.1:9092",
+  "group.id": "ruby-kafka-test",
+
+  # Disable automatic offsit commits, requiring the consumer to call commit on
+  # the consumer. Commits keep track of what messages have been processed to
+  # reduce replays of messages.
+  "enable.auto.commit": false,
+})
+
+# Initialize a topic with 8 partitions and 1 replica per partition. This is
+# only for testing, a replication factor of 1 is not generally recommended for
+# production.
+admin = Kafka::Admin.new(config)
+admin.create_topic("ruby_test_topic", 8, 1)
+admin.close
+
+@run = true
+trap("INT")  { @run = false }
+trap("TERM") { @run = false }
+
+4.times.map do |i|
+  # While librdkafka is thread safe, it doesn't make sense to have multiple
+  # threads polling a single consumer as they will end up just taking turns
+  # consuming the next available message.
+  #
+  # To get better coverage it is better to have a consumer per thread. Be aware
+  # that having more threads than partitions will cause some to sit idle
+  # waiting for a consumer to fail. At most there can be one consumer per
+  # partition for a topic actively receiving messages.
+  Thread.new do
+    con = Kafka::Consumer.new(config)
+    con.subscribe("ruby_test_topic")
+
+    while @run
+      con.poll(timeout: 500) do |msg|
+        # Requires Ruby 2.5. Most efficient way to convert from a millisecond
+        # timestamp to a Time.
+        ts = Time.at(0, msg.timestamp, :millisecond).utc
+
+        puts format("%2d: %4d %8d %s %s => %s", i, msg.partition, msg.offset, msg.key, ts, msg.payload)
+
+        con.commit(msg, async: true)
+      end
+    end
+
+    con.close
+  end
+end.map(&:join)