deimos-ruby 1.3.0.pre.beta5 → 1.4.0.pre.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5bf921e758c4cab2c5d35334e1f7dc99382ce414ce1c7492d42175f4c874ae54
-  data.tar.gz: f1d2c561854e12adf325ee3a6805f3ddac40f185fe5420932438e38ffee43688
+  metadata.gz: 8b4c2a6f41276ada5bf402ac01c6cf445a014a155bbcc6fe48e83bbb9aa1d8a2
+  data.tar.gz: 1041a4c5051900134f66893e86564c0ef896df269d0af9973aaf7f1f7cecb733
 SHA512:
-  metadata.gz: b9fe23ad5ecb545ad6a8743bfd5bfb735d9b7d672ae5dfee16d2456b02397c7f8ef3adc71a43a4043aa02974bfa1d6a7a60cd373e8f522e03eba739e877c82cb
-  data.tar.gz: df9ea2796a368360ff79ada8735060f7585408d7c59043b488a31bf9fcbb4e7bcd52017882f2f458b8b48d30ff8807e93c640820fef264d859bc1be329bb668e
+  metadata.gz: fdea52b89a1b788d9bdd5b6244d4f278139fb01e10b201720dd2afe052a418251a784d2756624cb3896d705016dbca0252206b27163a2c5e2c7eb200294b5979
+  data.tar.gz: 5df060deb523e4b4879f47a18034960afb905edde95da9ac98f7dd5f5e0a022e1e8be3b81c13685c26c6837ed3ef6505394607469b075b8e2b1068ddabf61bcb

data/CHANGELOG.md

@@ -7,15 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
-# [1.3.0-beta5] - 2020-01-14
-- Added `db_producer.insert` and `db_producer.process` metrics.
-
-# [1.3.0-beta4] - 2019-12-02
-- Fixed bug where by running `rake deimos:start` without
-  specifying a producer backend would crash.
-
-# [1.3.0-beta3] - 2019-11-26
-- Fixed bug in TestHelpers where key_decoder was not stubbed out.
+# [1.4.0-beta1] - 2019-11-22
+- Complete revamp of configuration method.
 
 # [1.3.0-beta2] - 2019-11-22
 - Fixed bug where consumers would require a key config in all cases

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.3.0.pre.beta4)
+    deimos-ruby (1.4.0.pre.beta1)
       avro-patches (~> 0.3)
       avro_turf (~> 0.8)
       phobos (~> 1.8.2.pre.beta2)
@@ -10,32 +10,6 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (5.2.0)
-      actionpack (= 5.2.0)
-      nio4r (~> 2.0)
-      websocket-driver (>= 0.6.1)
-    actionmailer (5.2.0)
-      actionpack (= 5.2.0)
-      actionview (= 5.2.0)
-      activejob (= 5.2.0)
-      mail (~> 2.5, >= 2.5.4)
-      rails-dom-testing (~> 2.0)
-    actionpack (5.2.0)
-      actionview (= 5.2.0)
-      activesupport (= 5.2.0)
-      rack (~> 2.0)
-      rack-test (>= 0.6.3)
-      rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    actionview (5.2.0)
-      activesupport (= 5.2.0)
-      builder (~> 3.1)
-      erubi (~> 1.4)
-      rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.0.3)
-    activejob (5.2.0)
-      activesupport (= 5.2.0)
-      globalid (>= 0.3.6)
     activemodel (5.2.0)
       activesupport (= 5.2.0)
     activerecord (5.2.0)
@@ -44,10 +18,6 @@ GEM
       arel (>= 9.0)
     activerecord-import (0.28.1)
       activerecord (>= 3.2)
-    activestorage (5.2.0)
-      actionpack (= 5.2.0)
-      activerecord (= 5.2.0)
-      marcel (~> 0.3.1)
     activesupport (5.2.0)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
@@ -62,24 +32,19 @@ GEM
     avro_turf (0.11.0)
       avro (>= 1.7.7, < 1.10)
       excon (~> 0.45)
-    builder (3.2.3)
     coderay (1.1.2)
     concurrent-ruby (1.1.3)
     concurrent-ruby-ext (1.1.3)
       concurrent-ruby (= 1.1.3)
-    crass (1.0.5)
     ddtrace (0.22.0)
       msgpack
     diff-lcs (1.3)
     digest-crc (0.4.1)
     dogstatsd-ruby (4.2.0)
-    erubi (1.9.0)
-    excon (0.71.0)
+    excon (0.69.1)
     exponential-backoff (0.0.4)
     ffi (1.11.1)
     formatador (0.2.5)
-    globalid (0.4.2)
-      activesupport (>= 4.2.0)
     guard (2.14.2)
       formatador (>= 0.2.4)
       listen (>= 2.7, < 4.0)
@@ -108,26 +73,13 @@ GEM
     logging (2.2.2)
       little-plugger (~> 1.1)
       multi_json (~> 1.10)
-    loofah (2.4.0)
-      crass (~> 1.0.2)
-      nokogiri (>= 1.5.9)
     lumberjack (1.0.12)
-    mail (2.7.1)
-      mini_mime (>= 0.1.1)
-    marcel (0.3.3)
-      mimemagic (~> 0.3.2)
     method_source (0.9.0)
-    mimemagic (0.3.3)
-    mini_mime (1.0.2)
-    mini_portile2 (2.4.0)
     minitest (5.11.3)
     msgpack (1.2.10)
     multi_json (1.14.1)
     mysql2 (0.5.2)
     nenv (0.3.0)
-    nio4r (2.5.2)
-    nokogiri (1.10.5)
-      mini_portile2 (~> 2.4.0)
     notiffany (0.1.1)
       nenv (~> 0.1)
       shellany (~> 0.0)
@@ -146,33 +98,6 @@ GEM
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
-    rack (2.0.7)
-    rack-test (1.1.0)
-      rack (>= 1.0, < 3)
-    rails (5.2.0)
-      actioncable (= 5.2.0)
-      actionmailer (= 5.2.0)
-      actionpack (= 5.2.0)
-      actionview (= 5.2.0)
-      activejob (= 5.2.0)
-      activemodel (= 5.2.0)
-      activerecord (= 5.2.0)
-      activestorage (= 5.2.0)
-      activesupport (= 5.2.0)
-      bundler (>= 1.3.0)
-      railties (= 5.2.0)
-      sprockets-rails (>= 2.0.0)
-    rails-dom-testing (2.0.3)
-      activesupport (>= 4.2.0)
-      nokogiri (>= 1.6)
-    rails-html-sanitizer (1.3.0)
-      loofah (~> 2.3)
-    railties (5.2.0)
-      actionpack (= 5.2.0)
-      activesupport (= 5.2.0)
-      method_source
-      rake (>= 0.8.7)
-      thor (>= 0.18.1, < 2.0)
     rainbow (3.0.0)
     rake (10.5.0)
     rb-fsevent (0.10.2)
@@ -207,22 +132,12 @@ GEM
     ruby-progressbar (1.10.1)
     ruby_dep (1.5.0)
     shellany (0.0.1)
-    sprockets (4.0.0)
-      concurrent-ruby (~> 1.0)
-      rack (> 1, < 3)
-    sprockets-rails (3.2.1)
-      actionpack (>= 4.0)
-      activesupport (>= 4.0)
-      sprockets (>= 3.0.0)
     sqlite3 (1.3.13)
     thor (0.20.3)
     thread_safe (0.3.6)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
     unicode-display_width (1.6.0)
-    websocket-driver (0.7.1)
-      websocket-extensions (>= 0.1.0)
-    websocket-extensions (0.1.4)
 
 PLATFORMS
   ruby
@@ -239,7 +154,6 @@ DEPENDENCIES
   guard-rubocop (~> 1)
   mysql2 (~> 0.5)
   pg (~> 1.1)
-  rails (~> 5.2)
   rake (~> 10)
   rspec (~> 3)
   rspec_junit_formatter (~> 0.3)

data/README.md

@@ -58,88 +58,7 @@ gem 'deimos-ruby', '~> 1.1'
 
 # Configuration
 
-To configure the gem, use `configure` in an initializer:
-
-```ruby
-Deimos.configure do |config|
-  # Configure logger
-  config.logger = Rails.logger
-
-  # Phobos settings
-  config.phobos_config_file = 'config/phobos.yml'
-  config.schema_registry_url = 'https://my-schema-registry.com'
-  config.seed_broker = 'my.seed.broker.0.net:9093,my.seed.broker.1.net:9093'
-  config.ssl_enabled = ENV['KAFKA_SSL_ENABLED']
-  if config.ssl_enabled
-    config.ssl_ca_cert = File.read(ENV['SSL_CA_CERT'])
-    config.ssl_client_cert = File.read(ENV['SSL_CLIENT_CERT'])
-    config.ssl_client_cert_key = File.read(ENV['SSL_CLIENT_CERT_KEY'])
-  end
-  
-  # Other settings
-
-  # Local path to find schemas, for publishing and testing consumers
-  config.schema_path = "#{Rails.root}/app/schemas"
-
-  # Default namespace for producers to use
-  config.producer_schema_namespace = 'com.deimos.my_app'
-  
-  # Prefix for all topics, e.g. environment name
-  config.producer_topic_prefix = 'myenv.'
-   
-  # Disable all producers - e.g. when doing heavy data lifting and events
-  # would be fired a different way
-  config.disable_producers = true 
-  
-  # Default behavior is to swallow uncaught exceptions and log to DataDog.
-  # Set this to true to instead raise all errors. Note that raising an error
-  # will ensure that the message cannot be processed - if there is a bad
-  # message which will always raise that error, your consumer will not
-  # be able to proceed past it and will be stuck forever until you fix
-  # your code.
-  config.reraise_consumer_errors = true
-
-  # Another way to handle errors is to set reraise_consumer_errors to false
-  # but to set a global "fatal error" block that determines when to reraise:
-  config.fatal_error do |exception, payload, metadata|
-    exception.is_a?(BadError)
-  end
-  # Another example would be to check the database connection and fail
-  # if the DB is down entirely.
-  
-  # Set to true to send consumer lag metrics
-  config.report_lag = %w(production staging).include?(Rails.env)
-  
-  # Change the default backend. See Backends, below.
-  config.backend = :db
-
-  # Database Backend producer configuration
- 
-  # Logger for DB producer
-  config.db_producer.logger = Logger.new('/db_producer.log')
-
-  # List of topics to print full messages for, or :all to print all
-  # topics. This can introduce slowdown since it needs to decode
-  # each message using the schema registry. 
-  config.db_producer.log_topics = ['topic1', 'topic2']
-
-  # List of topics to compact before sending, i.e. only send the
-  # last message with any given key in a batch. This is an optimization
-  # which mirrors what Kafka itself will do with compaction turned on
-  # but only within a single batch.  You can also specify :all to
-  # compact all topics.
-  config.db_producer.compact_topics = ['topic1', 'topic2']
-
-  # Configure the metrics provider (see below).
-  config.metrics = Deimos::Metrics::Mock.new({ tags: %w(env:prod my_tag:another_1) })
-
-  # Configure the tracing provider (see below).
-  config.tracer = Deimos::Tracing::Mock.new({service_name: 'my-service'})
-end
-```
-
-Note that the configuration options from Phobos (seed_broker and the SSL settings)
-can be removed from `phobos.yml` since Deimos will load them instead.
+For a full configuration reference, please see [the configuration docs ](docs/CONFIGURATION.md).
 
 # Producers
 
@@ -148,15 +67,6 @@ Producers will look like this:
 ```ruby
 class MyProducer < Deimos::Producer
 
-  # Can override default namespace.
-  namespace 'com.deimos.my-app-special'
-  topic 'MyApp.MyTopic'
-  schema 'MySchema'
-  key_config field: 'my_field' # see Kafka Message Keys, below
-  
-  # If config.schema_path is app/schemas, assumes there is a file in
-  # app/schemas/com/deimos/my-app-special/MySchema.avsc
-
   class << self
   
     # Optionally override the default partition key logic, which is to use
@@ -342,18 +252,6 @@ Here is a sample consumer:
 ```ruby
 class MyConsumer < Deimos::Consumer
 
-  # These are optional but strongly recommended for testing purposes; this
-  # will validate against a local schema file used as the reader schema,
-  # as well as being able to write tests against this schema.
-  # This is recommended since it ensures you are always getting the values 
-  # you expect.
-  schema 'MySchema'
-  namespace 'com.my-namespace'
-  # This directive works identically to the producer - see Kafka Keys, above.
-  # This only affects the `decode_key` method below. You need to provide
-  # `schema` and `namespace`, above, for this to work.
-  key_config field: :my_id 
-
   # Optionally overload this to consider a particular exception
   # "fatal" only for this consumer. This is considered in addition
   # to the global `fatal_error` configuration block. 
@@ -381,7 +279,7 @@ what downstream systems are doing with it) is causing it. If you do
 not continue on past this message, your consumer will essentially be
 stuck forever unless you take manual action to skip the offset.
 
-Use `config.reraise_consumer_errors = false` to swallow errors. You
+Use `config.consumers.reraise_errors = false` to swallow errors. You
 can use instrumentation to handle errors you receive. You can also
 specify "fatal errors" either via global configuration (`config.fatal_error`)
 or via overriding a method on an individual consumer (`def fatal_error`).
@@ -393,15 +291,17 @@ messages as an array and then process them together. This can improve
 consumer throughput, depending on the use case. Batch consumers behave like
 other consumers in regards to key and payload decoding, etc.
 
-To enable batch consumption, create a listener in `phobos.yml` and ensure that
-the `delivery` property is set to `inline_batch`. For example:
+To enable batch consumption, ensure that the `delivery` property is set to `inline_batch`. For example:
 
-```yaml
-listeners:
-  - handler: Consumers::MyBatchConsumer
-    topic: my_batched_topic
-    group_id: my_group_id
-    delivery: inline_batch
+```ruby
+Deimos.configure do
+  consumer do
+    class_name 'Consumers::MyBatchConsumer'
+    topic 'my_batched_topic'
+    group_id 'my_group_id'
+    delivery :inline_batch
+  end
+end
 ```
 
 Batch consumers must inherit from the Deimos::BatchConsumer class as in
@@ -410,11 +310,6 @@ this sample:
 ```ruby
 class MyBatchConsumer < Deimos::BatchConsumer
 
-  # See the Consumer sample in the previous section
-  schema 'MySchema'
-  namespace 'com.my-namespace'
-  key_config field: :my_id 
-
   def consume_batch(payloads, metadata)
     # payloads is an array of Avro-decoded hashes.
     # metadata is a hash that contains information like :keys and :topic.
@@ -441,10 +336,6 @@ An example would look like this:
 ```ruby
 class MyProducer < Deimos::ActiveRecordProducer
 
-  topic 'MyApp.MyTopic'
-  schema 'MySchema'
-  key_config field: 'my_field'
-  
   # The record class should be set on every ActiveRecordProducer.
   # By default, if you give the producer a hash, it will re-fetch the
   # record itself for use in the payload generation. This can be useful
@@ -479,7 +370,7 @@ MyProducer.send_events([{foo: 1}, {foo: 2}])
 
 You can disable producers globally or inside a block. Globally:
 ```ruby
-Deimos.config.disable_producers = true
+Deimos.config.producers.disabled = true
 ```
 
 For the duration of a block:
@@ -560,7 +451,7 @@ To enable this, first generate the migration to create the relevant tables:
     
 You can now set the following configuration:
 
-    config.publish_backend = :db
+    config.producers.backend = :db
 
 This will save all your Kafka messages to the `kafka_messages` table instead
 of immediately sending to Kafka. Now, you just need to call
@@ -615,9 +506,6 @@ A sample consumer would look as follows:
 
 ```ruby
 class MyConsumer < Deimos::ActiveRecordConsumer
-
-  schema 'MySchema'
-  key_config field: 'my_field'
   record_class Widget
 
   # Optional override of the way to fetch records based on payload and
@@ -665,7 +553,7 @@ Deimos includes some metrics reporting out the box. It ships with DataDog suppor
 The following metrics are reported:
 * `consumer_lag` - for each partition, the number of messages
   it's behind the tail of the partition (a gauge). This is only sent if
-  `config.report_lag` is set to true.
+  `config.consumers.report_lag` is set to true.
 * `handler` - a count of the number of messages received. Tagged
   with the following:
     * `topic:{topic_name}`
@@ -693,15 +581,10 @@ The following metrics are reported:
   with the database backend. Tagged with the topic that is waiting.
   Will send a value of 0 with no topics tagged if there are no messages
   waiting.
-* `db_producer.insert` - the number of messages inserted into the database
-  for publishing. Tagged with `topic:{topic_name}`
-* `db_producer.process` - the number of DB messages processed. Note that this
-  is *not* the same as the number of messages *published* if those messages
-  are compacted. Tagged with `topic:{topic_name}`
 
 ### Configuring Metrics Providers
 
-See the `# Configure Metrics Provider` section under [Configuration](#configuration)
+See the `metrics` field under [Configuration](CONFIGURATION.md).
 View all available Metrics Providers [here](lib/deimos/metrics/metrics_providers)
 
 ### Custom Metrics Providers
@@ -726,7 +609,7 @@ separate span for message consume logic.
 
 ### Configuring Tracing Providers
 
-See the `# Configure Tracing Provider` section under [Configuration](#configuration)
+See the `tracing` field under [Configuration](#configuration).
 View all available Tracing Providers [here](lib/deimos/tracing)
 
 ### Custom Tracing Providers
@@ -768,7 +651,7 @@ test_consume_message(MyConsumer,
 end
 
 # You can also pass a topic name instead of the consumer class as long
-# as the topic is configured in your phobos.yml configuration:
+# as the topic is configured in your Deimos configuration:
 test_consume_message('my-topic-name',
                     { 'some-payload' => 'some-value' }) do |payload, metadata|
       # do some expectation handling here

data/Rakefile

@@ -10,4 +10,4 @@ rescue LoadError # rubocop:disable Lint/HandleExceptions
   # no rspec available
 end
 
-import('./lib/tasks/deimos.rake')
+import('./lib/tasks/phobos.rake')

data/deimos-ruby.gemspec

@@ -33,7 +33,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('guard-rubocop', '~> 1')
   spec.add_development_dependency('mysql2', '~> 0.5')
   spec.add_development_dependency('pg', '~> 1.1')
-  spec.add_development_dependency('rails', '~> 5.2')
   spec.add_development_dependency('rake', '~> 10')
   spec.add_development_dependency('rspec', '~> 3')
   spec.add_development_dependency('rspec_junit_formatter', '~>0.3')

data/docs/CONFIGURATION.md

@@ -0,0 +1,200 @@
+# Configuration
+
+Deimos supports a succinct, readable syntax which uses
+pure Ruby to allow flexible configuration.
+
+You can access any configuration value via a simple `Deimos.config.whatever`.
+
+Nested configuration is denoted in simple dot notation:
+`kafka.ssl.enabled`. Headings below will follow the nested
+configurations.
+
+## Base Configuration
+Config name|Default|Description
+-----------|-------|-----------
+logger|`Logger.new(STDOUT)`|The logger that Deimos will use.
+phobos_logger|`Deimos.config.logger`|The logger passed to Phobos.
+metrics|`Deimos::Metrics::Mock.new`|The metrics backend use for reporting.
+tracer|`Deimos::Tracer::Mock.new`|The tracer backend used for debugging.
+
+## Defining Producers
+
+You can define a new producer thusly:
+```ruby
+Deimos.configure do
+  producer do
+    class_name 'MyProducer'
+    topic 'MyTopic'
+    schema 'MyTopicSchema'
+    namespace 'my.namespace'
+    key_config field: :id
+
+  # If config.schema.path is app/schemas, assumes there is a file in
+  # app/schemas/my/namespace/MyTopicSchema.avsc
+  end
+end
+```
+
+You can have as many `producer` blocks as you like to define more producers.
+|Config name|Default|Description
+-----------|-------|-----------
+class_name|nil|Class name of the producer class (subclass of `Deimos::Producer`.)
+topic|nil|Topic to produce to.
+schema|nil|Name of the schema to use to encode data before producing.
+namespace|nil|Namespace of the schema to use when finding it locally.
+key_config|nil|Configuration hash for message keys. See [Kafka Message Keys](../README.md#installation)
+
+## Defining Consumers
+
+Consumers are defined almost identically to producers:
+
+```ruby
+Deimos.configure do
+  consumer do
+    class_name 'MyConsumer'
+    topic 'MyTopic'
+    schema 'MyTopicSchema'
+    namespace 'my.namespace'
+    key_config field: :id
+
+  # If config.schema.path is app/schemas, assumes there is a file in
+  # app/schemas/my/namespace/MyTopicSchema.avsc
+  end
+end
+```
+
+In addition to the producer configs, you can define a number of overrides
+to the basic consumer configuration for each consumer. This is analogous to
+the `listener` config in `phobos.yml`.
+|Config name|Default|Description
+-----------|-------|-----------
+class_name|nil|Class name of the consumer class (subclass of `Deimos::Consumer`.)
+topic|nil|Topic to produce to.
+schema|nil|This is optional but strongly recommended for testing purposes; this will validate against a local schema file used as the reader schema, as well as being able to write tests against this schema. This is recommended since it ensures you are always getting the values  you expect.
+namespace|nil|Namespace of the schema to use when finding it locally.
+key_config|nil|Configuration hash for message keys. See [Kafka Message Keys](../README.md#installation)
+group_id|nil|ID of the consumer group.
+max_concurrency|1|Number of threads created for this listener. Each thread will behave as an independent consumer. They don't share any state.
+start_from_beginning|true|Once the consumer group has checkpointed its progress in the topic's partitions, the consumers will always start from the checkpointed offsets, regardless of config. As such, this setting only applies when the consumer initially starts consuming from a topic
+max_bytes_per_partition|512.kilobytes|Maximum amount of data fetched from a single partition at a time.
+min_bytes|1|Minimum number of bytes to read before returning messages from the server; if `max_wait_time` is reached, this is ignored.
+max_wait_time|5|Maximum duration of time to wait before returning messages from the server, in seconds.
+force_encoding|nil|Apply this encoding to the message payload. If blank it uses the original encoding. This property accepts values defined by the ruby Encoding class (https://ruby-doc.org/core-2.3.0/Encoding.html). Ex: UTF_8, ASCII_8BIT, etc.
+delivery|`:batch`|The delivery mode for the consumer. Possible values: `:message, :batch, :inline_batch`. See Phobos documentation for more details.
+session_timeout|300|Number of seconds after which, if a client hasn't contacted the Kafka cluster, it will be kicked out of the group.
+offset_commit_interval|10|Interval between offset commits, in seconds.
+offset_commit_threshold|0|Number of messages that can be processed before their offsets are committed. If zero, offset commits are not triggered by message processing
+heartbeat_interval|10|Interval between heartbeats; must be less than the session window.
+backoff|`(1000..60_000)`|Range representing the minimum and maximum number of milliseconds to back off after a consumer error.
+
+## Kafka Configuration
+Config name|Default|Description
+-----------|-------|-----------
+kafka.logger|`Deimos.config.logger`|Logger passed to RubyKafka.
+kafka.seed_brokers|`['localhost:9092']`|URL for the Kafka brokers.
+kafka.client_id|`phobos`|Identifier for this application.
+kafka.connect_timeout|15|The socket timeout for connecting to the broker, in seconds.
+kafka.socket_timeout|15|The socket timeout for reading and writing to the broker, in seconds.
+kafka.ssl.enabled|false|Whether SSL is enabled on the brokers.
+kafka.ssl.ca_cert|nil| A PEM encoded CA cert, a file path to the cert, or an Array of certs to use with an SSL connection.
+kafka.ssl.client_cert|nil|A PEM encoded client cert to use with an SSL connection, or a file path to the cert.
+kafka.ssl.client_cert_key|nil|A PEM encoded client cert key to use with an SSL connection.
+
+## Consumer Configuration
+
+These are top-level configuration settings, but they can be overridden
+by individual consumers.
+
+|Config name|Default|Description
+-----------|-------|-----------
+consumers.session_timeout|300|Number of seconds after which, if a client hasn't contacted the Kafka cluster, it will be kicked out of the group.
+consumers.offset_commit_interval|10|Interval between offset commits, in seconds.
+consumers.offset_commit_threshold|0|Number of messages that can be processed before their offsets are committed. If zero, offset commits are not triggered by message processing
+consumers.heartbeat_interval|10|Interval between heartbeats; must be less than the session window.
+consumers.backoff|`(1000..60_000)`|Range representing the minimum and maximum number of milliseconds to back off after a consumer error.
+consumers.reraise_errors|false|Default behavior is to swallow uncaught exceptions and log to the metrics provider. Set this to true to instead raise all errors. Note that raising an error will ensure that the message cannot be processed - if there is a bad message which will always raise that error, your consumer will not be able to proceed past it and will be stuck forever until you fix your code. See also the `fatal_error` configuration. This is automatically set to true when using the `TestHelpers` module in RSpec.
+consumers.report_lag|false|Whether to send the `consumer_lag` metric. This requires an extra thread per consumer.
+consumers.fatal_error|`proc { false }`|Block taking an exception, payload and metadata and returning true if this should be considered a fatal error and false otherwise. E.g. you can use this to always fail if the database is available. Not needed if reraise_errors is set to true.
+
+## Producer Configuration
+|Config name|Default|Description
+-----------|-------|-----------
+producers.ack_timeout|5|Number of seconds a broker can wait for replicas to acknowledge a write before responding with a timeout.
+producers.required_acks|1|Number of replicas that must acknowledge a write, or `:all` if all in-sync replicas must acknowledge.
+producers.max_retries|2|Number of retries that should be attempted before giving up sending messages to the cluster. Does not include the original attempt.
+producers.retry_backoff|1|Number of seconds to wait between retries.
+producers.max_buffer_size|10_000|Number of messages allowed in the buffer before new writes will raise `BufferOverflow` exceptions.
+producers.max_buffer_bytesize|10_000_000|Maximum size of the buffer in bytes. Attempting to produce messages when the buffer reaches this size will result in `BufferOverflow` being raised.
+producers.compression_codec|nil|Name of the compression codec to use, or nil if no compression should be performed. Valid codecs: `:snappy` and `:gzip`
+producers.compression_threshold|1|Number of messages that needs to be in a message set before it should be compressed. Note that message sets are per-partition rather than per-topic or per-producer.
+producers.max_queue_size|10_000|Maximum number of messages allowed in the queue. Only used for async_producer.
+producers.delivery_threshold|0|If greater than zero, the number of buffered messages that will automatically trigger a delivery. Only used for async_producer.
+producers.delivery_interval|0|if greater than zero, the number of seconds between automatic message deliveries. Only used for async_producer.
+producers.persistent_connections|false|Set this to true to keep the producer connection between publish calls. This can speed up subsequent messages by around 30%, but it does mean that you need to manually call sync_producer_shutdown before exiting, similar to async_producer_shutdown.
+producers.schema_namespace|nil|Default namespace for all producers. Can remain nil. Individual producers can override.
+producers.topic_prefix|nil|Add a prefix to all topic names. This can be useful if you're using the same Kafka broker for different environments that are producing the same topics.
+producers.disabled|false|Disable all actual message producing. Generally more useful to use the `disable_producers` method instead.
+producers.backend|`:kafka_async`|Currently can be set to `:db`, `:kafka`, or `:kafka_async`. If using Kafka directly, a good pattern is to set to async in your user-facing app, and sync in your consumers or delayed workers.
+
+## Schema Configuration
+|Config name|Default|Description
+-----------|-------|-----------
+schema.registry_url|`http://localhost:8081`|URL of the Confluent schema registry.
+schema.path|nil|Local path to find your schemas.
+
+## Database Producer Configuration
+|Config name|Default|Description
+-----------|-------|-----------
+db_producer.logger|`Deimos.config.logger`|Logger to use inside the DB producer.
+db_producer.log_topics|`[]`|List of topics to print full messages for, or `:all` to print all topics. This can introduce slowdown since it needs to decode each message using the schema registry. 
+db_producer.compact_topics|`[]`|List of topics to compact before sending, i.e. only send the last message with any given key in a batch. This is an optimization which mirrors what Kafka itself will do with compaction turned on but only within a single batch.  You can also specify `:all` to compact all topics.
+
+## Configuration Syntax
+
+Sample: 
+
+```ruby
+Deimos.configure do
+  logger Logger.new(STDOUT)
+  # Nested config field
+  kafka.seed_brokers ['my.kafka.broker:9092']
+
+  # Multiple nested config fields via block
+  consumers do
+    session_timeout 30
+    offset_commit_interval 10
+  end
+
+  # Define a new producer
+  producer do
+    class_name 'MyProducer'
+    topic 'MyTopic'
+    schema 'MyTopicSchema'
+    key_config field: :id
+  end
+
+  # Define another new producer
+  producer do
+    class_name 'AnotherProducer'
+    topic 'AnotherTopic'
+    schema 'AnotherSchema'
+    key_config plain: true
+  end
+
+  # Define a consumer
+  consumer do
+    class_name 'MyConsumer'
+    topic 'TopicToConsume'
+    schema 'ConsumerSchema'
+    key_config plain: true
+    # include Phobos / RubyKafka configs
+    start_from_beginning true
+    heartbeat_interval 10 
+  end
+
+end
+```
+
+Note that all blocks are evaluated in the context of the configuration object.
+If you're calling this inside another class or method, you'll need to save
+things you need to reference into local variables before calling `configure`.