racecar 1.0.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c69597ad113d51482fb3f8de7a3f6ab372ad9f08942ba528223e4f68affd017b
-  data.tar.gz: 2a5a97e179e7e7039dad1cd2a59bdd6ac61fa7e6313db707c5d74ee05c64b596
+  metadata.gz: e2c32338556d27bfcbae35df0758163a457d2c6e3f520141206c5a9c0124cc68
+  data.tar.gz: 43d9c42d0d483c269b15d417ef59e2985da3ca3945d34d6772f8dd262ccbfaf6
 SHA512:
-  metadata.gz: de0eec47d172624283b353e9ec858ea3c6af3582c7237a555d1049ef477805872f8dd0db27ac871aa218ba3f36b66369d00b4dea9bbb011a2bb534dc0f67d485
-  data.tar.gz: 25520f00f98c241db47ed4fedeae9ce65615604933c4178ae453756ee6d897f8c39c3b57232805f8a5f8106d06b37981456315b363c34b115d6aee8a738272f2
+  metadata.gz: 0467ac1cdefb6cad9870dd73b92f4a5a943b9f685ff3fc876b3f183d109ae3d29d7c2c7dffea8f31bca7c7b18565e5aba04d4865c94f2448a7228be175855a5b
+  data.tar.gz: e4ab43eb180995af916d447b006438b4a48cb808b29aabec52b455e246541a083192d1b560a957fa6f3ab7d5412dd12ab74aac49acc0b606c3df87cec90b93b6

data/.circleci/config.yml

@@ -0,0 +1,56 @@
+version: 2.1
+orbs:
+  ruby: circleci/ruby@0.1.2
+
+jobs:
+  build:
+    docker:
+      - image: circleci/ruby:2.6.3-stretch-node
+    executor: ruby/default
+    steps:
+      - checkout
+      - run:
+          name: Which bundler?
+          command: bundle -v
+      - ruby/bundle-install
+      - run: bundle exec rspec --exclude-pattern='spec/integration/*_spec.rb'
+  integration-tests:
+    docker:
+      - image: circleci/ruby:2.6.3-stretch-node
+      - image: wurstmeister/zookeeper
+      - image: wurstmeister/kafka:2.11-2.0.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9092
+          KAFKA_PORT: 9092
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.11-2.0.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9093
+          KAFKA_PORT: 9093
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.11-2.0.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9094
+          KAFKA_PORT: 9094
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+    executor: ruby/default
+    steps:
+      - checkout
+      - run:
+          name: Which bundler?
+          command: bundle -v
+      - ruby/bundle-install
+      - run: bundle exec rspec --pattern='spec/integration/*_spec.rb'
+
+workflows:
+  version: 2
+  test:
+    jobs:
+      - build
+      - integration-tests

data/.github/workflows/ci.yml

@@ -0,0 +1,61 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - '**'
+
+jobs:
+  unit-specs:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        ruby-version: ["2.5", "2.6"]
+
+    steps:
+    - uses: zendesk/checkout@v2
+    - name: Set up Ruby
+      uses: zendesk/setup-ruby@v1.58.0
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true
+    - name: Build and test with RSpec
+      run: bundle exec rspec --format documentation --require spec_helper --color --exclude-pattern='spec/integration/*_spec.rb'
+
+  integration-specs:
+    runs-on: ubuntu-latest
+
+    services:
+      zookeeper:
+        image: confluentinc/cp-zookeeper
+        ports:
+          - 2181:2181
+        env:
+          ZOOKEEPER_CLIENT_PORT: 2181
+
+      kafka:
+        image: confluentinc/cp-kafka
+        ports:
+          - 9092:9092
+          - 29092:29092
+        options: --health-cmd "kafka-topics --list --bootstrap-server=localhost:9092" --health-interval 10s --health-timeout 5s --health-retries 5
+        env:
+          KAFKA_BROKER_ID: 1
+          KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
+          KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+          KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+          KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+
+    steps:
+    - uses: zendesk/checkout@v2
+    - name: Set up Ruby
+      uses: zendesk/setup-ruby@v1.58.0
+      with:
+        ruby-version: 2.7
+        bundler-cache: true
+    - name: Build and test with RSpec
+      env:
+        RACECAR_BROKERS: localhost:9092
+      run: bundle exec rspec --format documentation --require spec_helper --color spec/integration/*_spec.rb

data/.gitignore

@@ -1,10 +1,9 @@
 /.bundle/
 /.yardoc
-/Gemfile.lock
 /_yardoc/
 /coverage/
 /doc/
 /pkg/
 /spec/reports/
 /tmp/
-/vendor/bundle/
+/vendor/bundle/

data/CHANGELOG.md

@@ -2,6 +2,48 @@
 
 ## Unreleased
 
+## racecar v2.2.0
+
+* [Racecar::ConsumerSet] **breaking change** `Racecar::ConsumerSet`'s functions `poll` and `batch_pall` expect the max wait values to be given in milliseconds. The defaults were using `config.max_wait_time`, which is in seconds. If you do not directly use `Racecar::ConsumerSet`, or always call its `poll` and `batch_poll` functions by specfiying the max wait time (the first argument), then this breaking change does not affect you. ([#214](https://github.com/zendesk/racecar/pull/214))
+
+## racecar v2.1.1
+
+* [Bugfix] Close RdKafka consumer in ConsumerSet#reset_current_consumer to prevent memory leak (#196)
+* [Bugfix] `poll`/`batch_poll` would not retry in edge cases and raise immediately. They still honor the `max_wait_time` setting, but might return no messages instead and only retry on their next call. ([#177](https://github.com/zendesk/racecar/pull/177))
+
+## racecar v2.1.0
+
+* Bump rdkafka to 0.8.0 (#191)
+
+## racecar v2.0.0
+
+* Replace `ruby-kafka` with `rdkafka-ruby` as the low-level library underneath Racecar (#91).
+* Fix `max_wait_time` usage (#179).
+* Removed config option `sasl_over_ssl`.
+* [Racecar::Consumer] Do not pause consuming partitions on exception.
+* [Racecar::Consumer] `topic`, `payload` and `key` are mandadory to method `produce`.
+* [Racecar::Consumer] `process_batch` retrieves an array of messages instead of batch object.
+* [Racecar::Consumer] Remove `offset_retention_time`.
+* [Racecar::Consumer] Allow providing `additional_config` for subscriptions.
+* [Racecar::Consumer] Provide access to `producer` and `consumer`.
+* [Racecar::Consumer] Enforce delivering messages with method `deliver!`.
+* [Racecar::Consumer] instead of raising when a partition EOF is reached, the result can be queried through `consumer.last_poll_read_partition_eof?`.
+* [Racecar::Config] Remove `offset_retention_time`, `connect_timeout` and `offset_commit_threshold`.
+* [Racecar::Config] Pass config to `rdkafka-ruby` via `producer` and `consumer`.
+* [Racecar::Config] Replace `max_fetch_queue_size` with `min_message_queue_size`.
+* [Racecar::Config] Add `synchronous_commits` to control blocking of `consumer.commit` (default `false`).
+* [Racecar::Config] Add `security_protocol` to control protocol between client and broker.
+* [Racecar::Config] SSL configuration via `ssl_ca_location`, `ssl_crl_location`, `ssl_keystore_location` and `ssl_keystore_password`.
+* [Racecar::Config] SASL configuration via `sasl_mechanism`, `sasl_kerberos_service_name`, `sasl_kerberos_principal`, `sasl_kerberos_kinit_cmd`, `sasl_kerberos_keytab`, `sasl_kerberos_min_time_before_relogin`, `sasl_username` and `sasl_password`.
+* [Instrumentation] `produce_message.racecar` sent whenever a produced message is queued. Payload includes `topic`, `key`, `value` and `create_time`.
+* [Instrumentation] `acknowledged_message.racecar` send whenever a produced message was successfully received by Kafka. Payload includes `offset` and `partition`, but no message details.
+* [Instrumentation] `rdkafka-ruby` does not yet provide instrumentation [rdkafka-ruby#54](https://github.com/appsignal/rdkafka-ruby/issues/54).
+* [Instrumentation] if processors define a `statistics_callback`, it will be called once every second for every subscription or producer connection. The first argument will be a Hash, for contents see [librdkafka STATISTICS.md](https://github.com/edenhill/librdkafka/blob/master/STATISTICS.md).
+* Add current directory to `$LOAD_PATH` only when `--require` option is used (#117).
+* Remove manual heartbeat support, see [Long-running message processing section in README](README.md#long-running-message-processing).
+* Rescue exceptions--then log and pass to `on_error`--at the outermost level of `exe/racecar`, so that exceptions raised outside `Cli.run` are not silently discarded (#186).
+* When exceptions with a `cause` are logged, recursively log the `cause` detail, separated by `--- Caused by: ---\n`.
+
 ## racecar v1.0.0
 
 Unchanged from v0.5.0.

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in racecar.gemspec

data/Gemfile.lock

@@ -0,0 +1,69 @@
+PATH
+  remote: .
+  specs:
+    racecar (2.1.1)
+      king_konf (~> 1.0.0)
+      rdkafka (~> 0.8.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.0.3.4)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.7)
+    diff-lcs (1.4.4)
+    dogstatsd-ruby (4.8.2)
+    ffi (1.13.1)
+    i18n (1.8.5)
+      concurrent-ruby (~> 1.0)
+    king_konf (1.0.0)
+    method_source (1.0.0)
+    mini_portile2 (2.5.0)
+    minitest (5.14.2)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rake (13.0.1)
+    rdkafka (0.8.1)
+      ffi (~> 1.9)
+      mini_portile2 (~> 2.1)
+      rake (>= 12.3)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.0)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.0)
+    thread_safe (0.3.6)
+    timecop (0.9.2)
+    tzinfo (1.2.8)
+      thread_safe (~> 0.1)
+    zeitwerk (2.4.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport (>= 4.0, < 6.1)
+  bundler (>= 1.13, < 3)
+  dogstatsd-ruby (>= 4.0.0, < 5.0.0)
+  pry
+  racecar!
+  rake (> 10.0)
+  rspec (~> 3.0)
+  timecop
+
+BUNDLED WITH
+   2.1.4

data/README.md

@@ -2,7 +2,7 @@
 
 Racecar is a friendly and easy-to-approach Kafka consumer framework. It allows you to write small applications that process messages stored in Kafka topics while optionally integrating with your Rails models.
 
-The framework is based on [ruby-kafka](https://github.com/zendesk/ruby-kafka), which, when used directly, can be a challenge: it's a flexible library with lots of knobs and options. Most users don't need that level of flexibility, though. Racecar provides a simple and intuitive way to build and configure Kafka consumers.
+The framework is based on [rdkafka-ruby](https://github.com/appsignal/rdkafka-ruby), which, when used directly, can be a challenge: it's a flexible library with lots of knobs and options. Most users don't need that level of flexibility, though. Racecar provides a simple and intuitive way to build and configure Kafka consumers.
 
 **NOTE:** Racecar requires Kafka 0.10 or higher.
 
@@ -19,6 +19,7 @@ The framework is based on [ruby-kafka](https://github.com/zendesk/ruby-kafka), w
     7. [Handling errors](#handling-errors)
     8. [Logging](#logging)
     9. [Operations](#operations)
+    10. [Upgrading from v1 to v2](#upgrading-from-v1-to-v2)
 3. [Development](#development)
 4. [Contributing](#contributing)
 5. [Support and Discussion](#support-and-discussion)
@@ -49,9 +50,7 @@ This will add a config file in `config/racecar.yml`.
 
 ## Usage
 
-Racecar is built for simplicity of development and operation. If you need more flexibility, it's quite straightforward to build your own Kafka consumer executables using [ruby-kafka](https://github.com/zendesk/ruby-kafka#consuming-messages-from-kafka) directly.
-
-First, a short introduction to the Kafka consumer concept as well as some basic background on Kafka.
+Racecar is built for simplicity of development and operation. First, a short introduction to the Kafka consumer concept as well as some basic background on Kafka.
 
 Kafka stores messages in so-called _partitions_ which are grouped into _topics_. Within a partition, each message gets a unique offset.
 
@@ -121,23 +120,23 @@ Note that once the consumer has started, it will commit the offsets it has proce
 
 #### Processing messages in batches
 
-If you want to process whole _batches_ of messages at a time, simply rename your `#process` method to `#process_batch`. The method will now be called with a "batch" object rather than a message:
+If you want to process whole _batches_ of messages at a time, simply rename your `#process` method to `#process_batch`. The method will now be called with an array of message objects:
 
 ```ruby
 class ArchiveEventsConsumer < Racecar::Consumer
   subscribes_to "events"
 
-  def process_batch(batch)
+  def process_batch(messages)
     file_name = [
-      batch.topic, # the topic this batch of messages came from.
-      batch.partition, # the partition this batch of messages came from.
-      batch.first_offset, # offset of the first message in the batch.
-      batch.last_offset, # offset of the last message in the batch.
+      messages.first.topic, # the topic this batch of messages came from.
+      messages.first.partition, # the partition this batch of messages came from.
+      messages.first.offset, # offset of the first message in the batch.
+      messages.last.offset, # offset of the last message in the batch.
     ].join("-")
 
     File.open(file_name, "w") do |file|
       # the messages in the batch.
-      batch.messages.each do |message|
+      messages.each do |message|
         file << message.value
       end
     end
@@ -155,23 +154,13 @@ Any headers set on the message will be available when consuming the message:
 message.headers #=> { "Header-A" => 42, ... }
 ```
 
-#### Heartbeats
+#### Long-running message processing
 
-In order to avoid your consumer being kicked out of its group during long-running message processing operations, it may be a good idea to periodically send so-called _heartbeats_ back to Kafka. This is done automatically for you after each message has been processed, but if the processing of a _single_ message takes a long time you may run into group stability issues.
+In order to avoid your consumer being kicked out of its group during long-running message processing operations, you'll need to let Kafka regularly know that the consumer is still healthy. There's two mechanisms in place to ensure that:
 
-If possible, intersperse `heartbeat` calls in between long-running operations in your consumer, e.g.
-
-```ruby
-def process(message)
-  long_running_op_one(message)
-
-  # Signals back to Kafka that we're still alive!
-  heartbeat
-
-  long_running_op_two(message)
-end
-```
+*Heartbeats:* They are automatically sent in the background and ensure the broker can still talk to the consumer. This will detect network splits, ungraceful shutdowns, etc.
 
+*Message Fetch Interval:* Kafka expects the consumer to query for new messages within this time limit. This will detect situations with slow IO or the consumer being stuck in an infinite loop without making actual progress. This limit applies to a whole batch if you do batch processing. Use `max_poll_interval` to increase the default 5 minute timeout, or reduce batching with `fetch_messages`.
 
 #### Tearing down resources when stopping
 
@@ -222,18 +211,20 @@ class GeoCodingConsumer < Racecar::Consumer
 
     # The `produce` method enqueues a message to be delivered after #process
     # returns. It won't actually deliver the message.
-    produce(JSON.dump(pageview), topic: "pageviews-with-country")
+    produce(JSON.dump(pageview), topic: "pageviews-with-country", key: pageview["id"])
   end
 end
 ```
 
+The `deliver!` method can be used to block until the broker received all queued published messages (according to the publisher ack settings). This will automatically being called in the shutdown procedure of a consumer.
+
 You can set message headers by passing a `headers:` option with a Hash of headers.
 
 ### Configuration
 
 Racecar provides a flexible way to configure your consumer in a way that feels at home in a Rails application. If you haven't already, run `bundle exec rails generate racecar:install` in order to generate a config file. You'll get a separate section for each Rails environment, with the common configuration values in a shared `common` section.
 
-**Note:** many of these configuration keys correspond directly to similarly named concepts in [ruby-kafka](https://github.com/zendesk/ruby-kafka); for more details on low-level operations, read that project's documentation.
+**Note:** many of these configuration keys correspond directly to similarly named concepts in [rdkafka-ruby](https://github.com/appsignal/rdkafka-ruby); for more details on low-level operations, read that project's documentation.
 
 It's also possible to configure Racecar using environment variables. For any given configuration key, there should be a corresponding environment variable with the prefix `RACECAR_`, in upper case. For instance, in order to configure the client id, set `RACECAR_CLIENT_ID=some-id` in the process in which the Racecar consumer is launched. You can set `brokers` by passing a comma-separated list, e.g. `RACECAR_BROKERS=kafka1:9092,kafka2:9092,kafka3:9092`.
 
@@ -263,8 +254,6 @@ end
 The consumers will checkpoint their positions from time to time in order to be able to recover from failures. This is called _committing offsets_, since it's done by tracking the offset reached in each partition being processed, and committing those offset numbers to the Kafka offset storage API. If you can tolerate more double-processing after a failure, you can increase the interval between commits in order to better performance. You can also do the opposite if you prefer less chance of double-processing.
 
 * `offset_commit_interval` – How often to save the consumer's position in Kafka. Default is every 10 seconds.
-* `offset_commit_threshold` – How many messages to process before forcing a checkpoint. Default is 0, which means there's no limit. Setting this to e.g. 100 makes the consumer stop every 100 messages to checkpoint its position.
-* `offset_retention_time` - How long committed offsets will be retained. Defaults to the broker setting.
 
 #### Timeouts & intervals
 
@@ -272,8 +261,9 @@ All timeouts are defined in number of seconds.
 
 * `session_timeout` – The idle timeout after which a consumer is kicked out of the group. Consumers must send heartbeats with at least this frequency.
 * `heartbeat_interval` – How often to send a heartbeat message to Kafka.
-* `pause_timeout` – How long to pause a partition for if the consumer raises an exception while processing a message. Default is to pause for 10 seconds. Set this to zero in order to disable automatic pausing of partitions.
-* `connect_timeout` – How long to wait when trying to connect to a Kafka broker. Default is 10 seconds.
+* `max_poll_interval` – The maximum time between two message fetches before the consumer is kicked out of the group. Put differently, your (batch) processing must finish earlier than this.
+* `pause_timeout` – How long to pause a partition for if the consumer raises an exception while processing a message. Default is to pause for 10 seconds. Set this to `0` in order to disable automatic pausing of partitions or to `-1` to pause indefinitely.
+* `pause_with_exponential_backoff` – Set to `true` if you want to double the `pause_timeout` on each consecutive failure of a particular partition.
 * `socket_timeout` – How long to wait when trying to communicate with a Kafka broker. Default is 30 seconds.
 * `max_wait_time` – How long to allow the Kafka brokers to wait before returning messages. A higher number means larger batches, at the cost of higher latency. Default is 1 second.
 
@@ -281,40 +271,37 @@ All timeouts are defined in number of seconds.
 
 Kafka is _really_ good at throwing data at consumers, so you may want to tune these variables in order to avoid ballooning your process' memory or saturating your network capacity.
 
-Racecar uses ruby-kafka under the hood, which fetches messages from the Kafka brokers in a background thread. This thread pushes fetch responses, possible containing messages from many partitions, into a queue that is read by the processing thread (AKA your code). The main way to control the fetcher thread is to control the size of those responses and the size of the queue.
+Racecar uses [rdkafka-ruby](https://github.com/appsignal/rdkafka-ruby) under the hood, which fetches messages from the Kafka brokers in a background thread. This thread pushes fetch responses, possible containing messages from many partitions, into a queue that is read by the processing thread (AKA your code). The main way to control the fetcher thread is to control the size of those responses and the size of the queue.
 
-* `max_bytes` — The maximum size of message sets returned from a single fetch request.
-* `max_fetch_queue_size` — The maximum number of fetch responses to keep in the queue. Once reached, the fetcher will back off until the queue gets back down under to limit.
+* `max_bytes` — Maximum amount of data the broker shall return for a Fetch request.
+* `min_message_queue_size` — The minimum number of messages in the local consumer queue.
 
-The memory usage limit is roughly estimated as `max_bytes * max_fetch_queue_size`, plus whatever your application uses.
+The memory usage limit is roughly estimated as `max_bytes * min_message_queue_size`, plus whatever your application uses.
 
 #### SSL encryption, authentication & authorization
 
-* `ssl_ca_cert` – A valid SSL certificate authority, as a string.
-* `ssl_ca_cert_file_path` - The path to a valid SSL certificate authority file.
-* `ssl_client_cert` – A valid SSL client certificate, as a string.
-* `ssl_client_cert_key` – A valid SSL client certificate key, as a string.
+* `security_protocol` – Protocol used to communicate with brokers (`:ssl`)
+* `ssl_ca_location` – File or directory path to CA certificate(s) for verifying the broker's key
+* `ssl_crl_location` – Path to CRL for verifying broker's certificate validity
+* `ssl_keystore_location` – Path to client's keystore (PKCS#12) used for authentication
+* `ssl_keystore_password` – Client's keystore (PKCS#12) password
+* `ssl_certificate_location` – Path to the certificate
+* `ssl_key_location` – Path to client's certificate used for authentication
+* `ssl_key_password` – Client's certificate password
 
 #### SASL encryption, authentication & authorization
 
-Racecar has support for using SASL to authenticate clients using either the GSSAPI or PLAIN mechanism.
+Racecar has support for using SASL to authenticate clients using either the GSSAPI or PLAIN mechanism either via plaintext or SSL connection.
 
-If using GSSAPI:
+* `security_protocol` – Protocol used to communicate with brokers (`:sasl_plaintext` `:sasl_ssl`)
+* `sasl_mechanism` – SASL mechanism to use for authentication (`GSSAPI` `PLAIN` `SCRAM-SHA-256` `SCRAM-SHA-512`)
 
-* `sasl_gssapi_principal` – The GSSAPI principal.
-* `sasl_gssapi_keytab` – Optional GSSAPI keytab.
-
-If using PLAIN:
-
-* `sasl_plain_authzid` – The authorization identity to use.
-* `sasl_plain_username` – The username used to authenticate.
-* `sasl_plain_password` – The password used to authenticate.
-
-If using SCRAM:
-
-* `sasl_scram_username` – The username used to authenticate.
-* `sasl_scram_password` – The password used to authenticate.
-* `sasl_scram_mechanism` – The SCRAM mechanism to use, either `sha256` or `sha512`.
+* `sasl_kerberos_principal` – This client's Kerberos principal name
+* `sasl_kerberos_kinit_cmd` – Full kerberos kinit command string, `%{config.prop.name}` is replaced by corresponding config object value, `%{broker.name}` returns the broker's hostname
+* `sasl_kerberos_keytab` – Path to Kerberos keytab file. Uses system default if not set
+* `sasl_kerberos_min_time_before_relogin` – Minimum time in milliseconds between key refresh attempts
+* `sasl_username` – SASL username for use with the PLAIN and SASL-SCRAM-.. mechanism
+* `sasl_password` – SASL password for use with the PLAIN and SASL-SCRAM-.. mechanism
 
 #### Producing messages
 
@@ -324,7 +311,7 @@ These settings are related to consumers that _produce messages to Kafka_.
 
 #### Datadog monitoring
 
-Racecar supports configuring ruby-kafka's [Datadog](https://www.datadoghq.com/) monitoring integration. If you're running a normal Datadog agent on your host, you just need to set `datadog_enabled` to `true`, as the rest of the settings come with sane defaults.
+Racecar supports [Datadog](https://www.datadoghq.com/) monitoring integration. If you're running a normal Datadog agent on your host, you just need to set `datadog_enabled` to `true`, as the rest of the settings come with sane defaults.
 
 * `datadog_enabled` – Whether Datadog monitoring is enabled (defaults to `false`).
 * `datadog_host` – The host running the Datadog agent.
@@ -332,6 +319,9 @@ Racecar supports configuring ruby-kafka's [Datadog](https://www.datadoghq.com/)
 * `datadog_namespace` – The namespace to use for Datadog metrics.
 * `datadog_tags` – Tags that should always be set on Datadog metrics.
 
+#### Consumers Without Rails ####
+
+By default, if Rails is detected, it will be automatically started when the consumer is started. There are cases where you might not want or need Rails. You can pass the `--without-rails` option when starting the consumer and Rails won't be started.
 
 ### Testing consumers
 
@@ -474,7 +464,7 @@ Racecar.config.on_error do |exception, info|
 end
 ```
 
-It is highly recommended that you set up an error handler.
+It is highly recommended that you set up an error handler. Please note that the `info` object contains different keys and values depending on whether you are using `process` or `process_batch`. See the `instrumentation_payload` object in the `process` and `process_batch` methods in the `Runner` class for the complete list.
 
 
 ### Logging
@@ -491,10 +481,17 @@ In order to gracefully shut down a Racecar consumer process, send it the `SIGTER
 In order to introspect the configuration of a consumer process, send it the `SIGUSR1` signal. This will make Racecar print its configuration to the standard error file descriptor associated with the consumer process, so you'll need to know where that is written to.
 
 
+### Upgrading from v1 to v2
+
+In order to safely upgrade from Racecar v1 to v2, you need to completely shut down your consumer group before starting it up again with the v2 Racecar dependency. In general, you should avoid rolling deploys for consumers groups, so it is likely the case that this will just work for you, but it's a good idea to check first.
+
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
+The integration tests run against a Kafka instance that is not automatically started from within `rspec`. You can set one up using the provided `docker-compose.yml` by running `docker-compose up`.
+
 
 ## Contributing
 
@@ -503,9 +500,9 @@ Bug reports and pull requests are welcome on [GitHub](https://github.com/zendesk
 
 ## Support and Discussion
 
-If you've discovered a bug, please file a [Github issue](https://github.com/zendesk/racecar/issues/new), and make sure to include all the relevant information, including the version of Racecar, ruby-kafka, and Kafka that you're using.
+If you've discovered a bug, please file a [Github issue](https://github.com/zendesk/racecar/issues/new), and make sure to include all the relevant information, including the version of Racecar, rdkafka-ruby, and Kafka that you're using.
 
-If you have other questions, or would like to discuss best practises, how to contribute to the project, or any other ruby-kafka related topic, [join our Slack team](https://ruby-kafka-slack.herokuapp.com/)!
+If you have other questions, or would like to discuss best practises, or how to contribute to the project, [join our Slack team](https://ruby-kafka-slack.herokuapp.com/)!
 
 
 ## Copyright and license