racecar 0.2.1 → 0.3.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fbf8129bd78b4936e7cf40832517e6619a7f0e17
-  data.tar.gz: 14f1432deab3e9f29236715e33f14d521e90ba42
+  metadata.gz: 21f3ecc4525a97eace384c5702c7f37939920292
+  data.tar.gz: 6bf40c5f643a9f0eb8fe6a962793a04e020becd4
 SHA512:
-  metadata.gz: 740d45ad33ae89a961338fa08125f9f7365ecb1b5f3a6d45732a43574a6c44e97432ceeed43325df32d713d2d3ec42d258eb2cc44b87319385df72305317a589
-  data.tar.gz: bb2b1b4eca7e542aaa53067d2adb6219eac9092d3f5a01cf51e9c67c33257f0ec83a6ea26955aa7d8a28105ede42d2c9400173b885552743eb1e1e460a4e3345
+  metadata.gz: 17a690c3206dc413c8795b36d2a8d01afbab74bbcd48a2c0e1d64bb8a662ae2d41516a4446d140967d5266e1bc5755b4e0814f269fc6c6f6dc1892eac7e05ed1
+  data.tar.gz: c811af7f6f5aa7cdd3953e1e954964ad6903e7740600e9ae16e5b8b2ef27234116b48c60fa6851d9166b157a07f1e44b851c310fd1a784a9717728d8218f43fc

data/.gitignore

@@ -7,3 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+/vendor/bundle/

data/README.md

@@ -4,6 +4,21 @@ Introducing Racecar, your friendly and easy-to-approach Kafka consumer framework
 
 Using [ruby-kafka](https://github.com/zendesk/ruby-kafka) directly can be a challenge: it's a flexible library with lots of knobs and options. Most users don't need that level of flexibility, though. Racecar provides a simple and intuitive way to build and configure Kafka consumers that optionally integrates seemlessly with Rails.
 
+## Table of content
+
+1. [Installation](#installation)
+2. [Usage](#usage)
+    1. [Creating consumers](#creating-consumers)
+    2. [Running consumers](#running-consumers)
+    3. [Configuration](#configuration)
+    4. [Testing consumers](#testing-consumers)
+    5. [Deploying consumers](#deploying-consumers)
+    6. [Operations](#operations)
+3. [Development](#development)
+4. [Contributing](#contributing)
+5. [Copyright and license](#copyright-and-license)
+
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -98,31 +113,96 @@ subscribes_to "some-topic", start_from_beginning: false
 
 Note that once the consumer has started, it will commit the offsets it has processed until and in the future will resume from those.
 
+#### Processing messages in batches
+
+If you want to process whole _batches_ of messages at a time, simply rename your `#process` method to `#process_batch`. The method will now be called with a "batch" object rather than a message:
+
+```ruby
+class ArchiveEventsConsumer < Racecar::Consumer
+  subscribes_to "events"
+
+  def process_batch(batch)
+    file_name = [
+      batch.topic, # the topic this batch of messages came from.
+      batch.partition, # the partition this batch of messages came from.
+      batch.first_offset, # offset of the first message in the batch.
+      batch.last_offset, # offset of the last message in the batch.
+    ].join("-")
+
+    File.open(file_name, "w") do |file|
+      # the messages in the batch.
+      batch.messages.each do |message|
+        file << message.value
+      end
+    end
+  end
+end
+```
+
+An important detail is that, if an exception is raised while processing a batch, the _whole batch_ is re-processed.
+
 ### Running consumers
 
 Racecar is first and foremost an executable _consumer runner_. The `racecar` executable takes as argument the name of the consumer class that should be run. Racecar automatically loads your Rails application before starting, and you can load any other library you need by passing the `--require` flag, e.g.
 
     $ bundle exec racecar --require dance_moves TapDanceConsumer
 
+The first time you execute `racecar` with a consumer class a _consumer group_ will be created with a group id derived from the class name (this can be configured). If you start `racecar` with the same consumer class argument multiple times, the processes will join the existing group – even if you start them on other nodes. You will typically want to have at least two consumers in each of your groups – preferably on separate nodes – in order to deal with failures.
+
 ### Configuration
 
 Racecar provides a flexible way to configure your consumer in a way that feels at home in a Rails application. If you haven't already, run `bundle exec rails generate racecar:install` in order to generate a config file. You'll get a separate section for each Rails environment, with the common configuration values in a shared `common` section.
 
-The possible configuration keys are:
+**Note:** many of these configuration keys correspond directly to similarly named concepts in [ruby-kafka](https://github.com/zendesk/ruby-kafka); for more details on low-level operations, read that project's documentation.
 
-* `brokers` (_optional_) – A list of Kafka brokers in the cluster that you're consuming from. Defaults to `localhost` on port 9092, the default Kafka port.
-* `client_id` (_optional_) – A string used to identify the client in logs and metrics.
-* `group_id_prefix` (_optional_) – A prefix used when generating consumer group names. For instance, if you set the prefix to be `kevin.` and your consumer class is named `BaconConsumer`, the resulting consumer group will be named `kevin.bacon_consumer`.
-* `offset_commit_interval` (_optional_) – How often to save the consumer's position in Kafka.
-* `heartbeat_interval` (_optional_) – How often to send a heartbeat message to Kafka.
-* `pause_timeout` (_optional_) – How long to pause a partition for if the consumer raises an exception while processing a message.
-* `connect_timeout` (_optional_) – How long to wait when trying to connect to a Kafka broker. Default is 10 seconds.
-* `socket_timeout` (_optional_) – How long to wait when trying to communicate with a Kafka broker. Default is 30 seconds.
-* `max_wait_time` (_optional_) – How long to allow the Kafka brokers to wait before returning messages. A higher number means larger batches, at the cost of higher latency. Default is 5 seconds.
+It's also possible to configure Racecar using environment variables. For any given configuration key, there should be a corresponding environment variable with the prefix `RACECAR_`, in upper case. For instance, in order to configure the client id, set `RACECAR_CLIENT_ID=some-id` in the process in which the Racecar consumer is launched. You can set `brokers` by passing a comma-separated list, e.g. `RACECAR_BROKERS=kafka1:9092,kafka2:9092,kafka3:9092`.
 
-Note that many of these configuration keys correspond directly with similarly named concepts in [ruby-kafka](https://github.com/zendesk/ruby-kafka) for more details on low-level operations, read that project's documentation.
+#### Basic configuration
+
+* `brokers` – A list of Kafka brokers in the cluster that you're consuming from. Defaults to `localhost` on port 9092, the default Kafka port.
+* `client_id` – A string used to identify the client in logs and metrics.
+* `group_id` – The group id to use for a given group of consumers. Note that this _must_ be different for each consumer class. If left blank a group id is generated based on the consumer class name.
+* `group_id_prefix` – A prefix used when generating consumer group names. For instance, if you set the prefix to be `kevin.` and your consumer class is named `BaconConsumer`, the resulting consumer group will be named `kevin.bacon_consumer`.
+
+#### Consumer checkpointing
+
+The consumers will checkpoint their positions from time to time in order to be able to recover from failures. This is called _committing offsets_, since it's done by tracking the offset reached in each partition being processed, and committing those offset numbers to the Kafka offset storage API. If you can tolerate more double-processing after a failure, you can increase the interval between commits in order to better performance. You can also do the opposite if you prefer less chance of double-processing.
+
+* `offset_commit_interval` – How often to save the consumer's position in Kafka. Default is every 10 seconds.
+* `offset_commit_threshold` – How many messages to process before forcing a checkpoint. Default is 0, which means there's no limit. Setting this to e.g. 100 makes the consumer stop every 100 messages to checkpoint its position.
+
+#### Timeouts & intervals
+
+All timeouts are defined in number of seconds.
+
+* `heartbeat_interval` – How often to send a heartbeat message to Kafka.
+* `pause_timeout` – How long to pause a partition for if the consumer raises an exception while processing a message.
+* `connect_timeout` – How long to wait when trying to connect to a Kafka broker. Default is 10 seconds.
+* `socket_timeout` – How long to wait when trying to communicate with a Kafka broker. Default is 30 seconds.
+* `max_wait_time` – How long to allow the Kafka brokers to wait before returning messages. A higher number means larger batches, at the cost of higher latency. Default is 5 seconds.
+
+#### SSL encryption, authentication & authorization
+
+* `ssl_ca_cert` – A valid SSL certificate authority, as a string.
+* `ssl_ca_cert_file_path` - The path to a valid SSL certificate authority file.
+* `ssl_client_cert` – A valid SSL client certificate, as a string.
+* `ssl_client_cert_key` – A valid SSL client certificate key, as a string.
+
+#### SASL encryption, authentication & authorization
+
+Racecar has support for using SASL to authenticate clients using either the GSSAPI or PLAIN mechanism.
+
+If using GSSAPI:
+
+* `sasl_gssapi_principal` – The GSSAPI principal
+* `sasl_gssapi_keytab` – Optional GSSAPI keytab.
+
+If using PLAIN:
+
+* `sasl_plain_authzid` – The authorization identity to use.
+* `sasl_plain_username` – The username used to authenticate.
+* `sasl_plain_password` – The password used to authenticate.
 
-It's also possible to configure Racecar using environment variables. For any given configuration key, there should be a corresponding environment variable with the prefix `RACECAR_`, in upper case. For instance, in order to configure the client id, set `RACECAR_CLIENT_ID=some-id` in the process in which the Racecar consumer is launched. You can set `brokers` by passing a comma-separated list, e.g. `RACECAR_BROKERS=kafka1:9092,kafka2:9092,kafka3:9092`.
 
 ### Testing consumers
 
@@ -158,6 +238,30 @@ describe CreateContactsConsumer do
 end
 ```
 
+
+### Deploying consumers
+
+If you're already deploying your Rails application using e.g. [Capistrano](http://capistranorb.com/), all you need to do to run your Racecar consumers in production is to have some _process supervisor_ start the processes and manage them for you.
+
+[Foreman](https://ddollar.github.io/foreman/) is a very straightford tool for interfacing with several process supervisor systems. You define your process types in a Procfile, e.g.
+
+```
+racecar-process-payments: bundle exec racecar ProcessPaymentsConsumer
+racecar-resize-images: bundle exec racecar ResizeImagesConsumer
+```
+
+If you've ever used Heroku you'll recognize the format – indeed, deploying to Heroku should just work if you add Racecar invocations to your Procfile.
+
+With Foreman, you can easily run these processes locally by executing `foreman run`; in production you'll want to _export_ to another process management format such as Upstart or Runit. [capistrano-foreman](https://github.com/hyperoslo/capistrano-foreman) allows you to do this with Capistrano.
+
+
+### Operations
+
+In order to gracefully shut down a Racecar consumer process, send it the `SIGTERM` signal. Most process supervisors such as Runit and Kubernetes send this signal when shutting down a process, so using those systems will make things easier.
+
+In order to introspect the configuration of a consumer process, send it the `SIGUSR1` signal. This will make Racecar print its configuration to the standard error file descriptor associated with the consumer process, so you'll need to know where that is written to.
+
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
@@ -170,7 +274,7 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/zendes
 
 ## Copyright and license
 
-Copyright 2017 Zendesk
+Copyright 2017 Daniel Schierbeck & Zendesk
 
 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.
 

data/examples/batch_consumer.rb

@@ -0,0 +1,9 @@
+class BatchConsumer < Racecar::Consumer
+  subscribes_to "messages", start_from_beginning: false
+
+  def process_batch(batch)
+    batch.messages.each do |message|
+      puts message.value
+    end
+  end
+end

data/lib/racecar/cli.rb

@@ -1,5 +1,5 @@
 require "optparse"
-require "racecar/config_loader"
+require "racecar/rails_config_file_loader"
 
 module Racecar
   module Cli
@@ -24,7 +24,7 @@ module Racecar
 
       puts "=> Starting Racecar consumer #{consumer_name}..."
 
-      ConfigLoader.load!
+      RailsConfigFileLoader.load!
 
       # Find the consumer class by name.
       consumer_class = Kernel.const_get(consumer_name)

data/lib/racecar/config.rb

@@ -7,8 +7,10 @@ module Racecar
     ALLOWED_KEYS = %w(
       brokers
       client_id
+
       offset_commit_interval
       offset_commit_threshold
+
       heartbeat_interval
       pause_timeout
       connect_timeout
@@ -16,9 +18,21 @@ module Racecar
       group_id_prefix
       group_id
       subscriptions
-      error_handler
       max_wait_time
+
+      error_handler
       log_to_stdout
+
+      ssl_ca_cert
+      ssl_ca_cert_file_path
+      ssl_client_cert
+      ssl_client_cert_key
+
+      sasl_gssapi_principal
+      sasl_gssapi_keytab
+      sasl_plain_authzid
+      sasl_plain_username
+      sasl_plain_password
     )
 
     REQUIRED_KEYS = %w(
@@ -70,6 +84,12 @@ module Racecar
       load_env!
     end
 
+    def inspect
+      ALLOWED_KEYS
+        .map {|key| [key, get(key).inspect].join(" = ") }
+        .join("\n")
+    end
+
     def validate!
       REQUIRED_KEYS.each do |key|
         if send(key).nil?
@@ -99,6 +119,10 @@ module Racecar
       load(data)
     end
 
+    def get(key)
+      public_send(key)
+    end
+
     def set(key, value)
       unless ALLOWED_KEYS.include?(key.to_s)
         raise ConfigError, "unknown configuration key `#{key}`"
@@ -148,6 +172,8 @@ module Racecar
       loader.integer(:connect_timeout)
       loader.integer(:socket_timeout)
       loader.integer(:max_wait_time)
+
+      loader.validate!
     end
   end
 end

data/lib/racecar/ctl.rb

@@ -1,5 +1,5 @@
 require "optparse"
-require "racecar/config_loader"
+require "racecar/rails_config_file_loader"
 
 module Racecar
   class Ctl
@@ -46,7 +46,7 @@ module Racecar
         raise Racecar::Error, "no message value specified"
       end
 
-      ConfigLoader.load!
+      RailsConfigFileLoader.load!
 
       Racecar.config.validate!
 

data/lib/racecar/env_loader.rb

@@ -3,6 +3,7 @@ module Racecar
     def initialize(env, config)
       @env = env
       @config = config
+      @loaded_keys = []
     end
 
     def string(name)
@@ -23,6 +24,16 @@ module Racecar
       set(name) {|value| value.split(",") }
     end
 
+    def validate!
+      # Make sure the user hasn't made a typo and added a key we don't know
+      # about.
+      @env.keys.grep(/^RACECAR_/).each do |key|
+        unless @loaded_keys.include?(key)
+          raise ConfigError, "unknown config variable #{key}"
+        end
+      end
+    end
+
     private
 
     def set(name)
@@ -31,6 +42,7 @@ module Racecar
       if @env.key?(key)
         value = yield @env.fetch(key)
         @config.set(name, value)
+        @loaded_keys << key
       end
     end
   end

data/lib/racecar/{config_loader.rb → rails_config_file_loader.rb}

@@ -1,5 +1,5 @@
 module Racecar
-  module ConfigLoader
+  module RailsConfigFileLoader
     def self.load!
       config_file = "config/racecar.yml"
 

data/lib/racecar/runner.rb

@@ -15,6 +15,9 @@ module Racecar
         logger: logger,
         connect_timeout: config.connect_timeout,
         socket_timeout: config.socket_timeout,
+        ssl_ca_cert: config.ssl_ca_cert,
+        ssl_client_cert: config.ssl_client_cert,
+        ssl_client_cert_key: config.ssl_client_cert_key,
       )
 
       consumer = kafka.consumer(
@@ -29,6 +32,9 @@ module Racecar
       trap("INT") { consumer.stop }
       trap("TERM") { consumer.stop }
 
+      # Print the consumer config to STDERR on USR1.
+      trap("USR1") { $stderr.puts config.inspect }
+
       config.subscriptions.each do |subscription|
         consumer.subscribe(
           subscription.topic,
@@ -38,8 +44,16 @@ module Racecar
       end
 
       begin
-        consumer.each_message(max_wait_time: config.max_wait_time) do |message|
-          processor.process(message)
+        if processor.respond_to?(:process)
+          consumer.each_message(max_wait_time: config.max_wait_time) do |message|
+            processor.process(message)
+          end
+        elsif processor.respond_to?(:process_batch)
+          consumer.each_batch(max_wait_time: config.max_wait_time) do |batch|
+            processor.process_batch(batch)
+          end
+        else
+          raise NotImplementedError, "Consumer class must implement process or process_batch method"
         end
       rescue Kafka::ProcessingError => e
         @logger.error "Error processing partition #{e.topic}/#{e.partition} at offset #{e.offset}"

data/lib/racecar/version.rb

@@ -1,3 +1,3 @@
 module Racecar
-  VERSION = "0.2.1"
+  VERSION = "0.3.0.beta1"
 end

data/racecar.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "ruby-kafka", "~> 0.3"
+  spec.add_runtime_dependency "ruby-kafka", "~> 0.4"
 
   spec.add_development_dependency "bundler", "~> 1.13"
   spec.add_development_dependency "rake", "~> 10.0"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0.beta1
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-07-17 00:00:00.000000000 Z
+date: 2017-08-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-kafka
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.4'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -85,6 +85,7 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- examples/batch_consumer.rb
 - examples/cat_consumer.rb
 - exe/racecar
 - exe/racecarctl
@@ -95,10 +96,10 @@ files:
 - lib/racecar.rb
 - lib/racecar/cli.rb
 - lib/racecar/config.rb
-- lib/racecar/config_loader.rb
 - lib/racecar/consumer.rb
 - lib/racecar/ctl.rb
 - lib/racecar/env_loader.rb
+- lib/racecar/rails_config_file_loader.rb
 - lib/racecar/runner.rb
 - lib/racecar/version.rb
 - racecar.gemspec
@@ -117,9 +118,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.4.5.1