racecar 0.3.5 → 0.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 227816cc1a2d86b97ae477dba46ef474c18b48e1
-  data.tar.gz: 6779b99acc433787743ac1e45963be0931458f11
+  metadata.gz: 296b4a32eb01dc8846049c0809af8bf34cf8c1f5
+  data.tar.gz: 0ec2243f458fbcd25a399d0920743b402a192886
 SHA512:
-  metadata.gz: 84f6a4fa70d0e680d17d4a6418c752353443263444528fd0d37a85e89b4005637627243b0150486a85b0439b8e1dd3b748e8f2a6d9296ee3c041f40de3b3cc03
-  data.tar.gz: 8d6262f0499bb2aa84d8beada1710fd32a97fad17aa8853f07cc90d38d739c1f65a5f40a9e4c0a409c9ca2e3f7a60eade093a0ed9d76569716e37d45a72b14d5
+  metadata.gz: 9c902215fbae7a6a9bf06bc3f1e68194b016f01283e3a13e0ba7e1949c3c902630aaedf9a3776806867219a0ce1f8097ca394e8ea3fd5500f2a4ba1ee58ab96c
+  data.tar.gz: 324330b9f6efeda4b769817bc54783f95d699f103121c117692b0d4b32e89373575081c7f1edad43dfb625bff3b95957f0245a96c5bc69c6daf0b91daa3673d1

data/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+## racecar v0.3.6
+
+* Allow producing messages (alpha).
+
 ## racecar v0.3.5
 
 * Instrument using ActiveSupport::Notifications (#43).

data/README.md

@@ -10,12 +10,13 @@ The framework is based on [ruby-kafka](https://github.com/zendesk/ruby-kafka), w
 2. [Usage](#usage)
     1. [Creating consumers](#creating-consumers)
     2. [Running consumers](#running-consumers)
-    3. [Configuration](#configuration)
-    4. [Testing consumers](#testing-consumers)
-    5. [Deploying consumers](#deploying-consumers)
-    6. [Handling errors](#handling-errors)
-    7. [Logging](#logging)
-    8. [Operations](#operations)
+    3. [Producing messages](#producing-messages)
+    4. [Configuration](#configuration)
+    5. [Testing consumers](#testing-consumers)
+    6. [Deploying consumers](#deploying-consumers)
+    7. [Handling errors](#handling-errors)
+    8. [Logging](#logging)
+    9. [Operations](#operations)
 3. [Development](#development)
 4. [Contributing](#contributing)
 5. [Support and Discussion](#support-and-discussion)
@@ -174,6 +175,31 @@ Racecar is first and foremost an executable _consumer runner_. The `racecar` exe
 
 The first time you execute `racecar` with a consumer class a _consumer group_ will be created with a group id derived from the class name (this can be configured). If you start `racecar` with the same consumer class argument multiple times, the processes will join the existing group – even if you start them on other nodes. You will typically want to have at least two consumers in each of your groups – preferably on separate nodes – in order to deal with failures.
 
+### Producing messages
+
+**WARNING:** This is an alpha feature, and could cause weird and unpredictable errors. Use with caution.
+
+Consumers can produce messages themselves, allowing for powerful stream processing applications that transform and filter message streams. The API for this is simple:
+
+```ruby
+class GeoCodingConsumer < Racecar::Consumer
+  subscribes_to "pageviews"
+
+  def process(message)
+    pageview = JSON.parse(message.value)
+    ip_address = pageview.fetch("ip_address")
+
+    country = GeoCode.country(ip_address)
+
+    # Enrich the original message:
+    pageview["country"] = country
+
+    # The `produce` method enqueues a message to be delivered after #process
+    # returns. It won't actually deliver the message.
+    produce(JSON.dump(pageview), topic: "pageviews-with-country")
+  end
+end
+```
 
 ### Configuration
 
@@ -183,6 +209,15 @@ Racecar provides a flexible way to configure your consumer in a way that feels a
 
 It's also possible to configure Racecar using environment variables. For any given configuration key, there should be a corresponding environment variable with the prefix `RACECAR_`, in upper case. For instance, in order to configure the client id, set `RACECAR_CLIENT_ID=some-id` in the process in which the Racecar consumer is launched. You can set `brokers` by passing a comma-separated list, e.g. `RACECAR_BROKERS=kafka1:9092,kafka2:9092,kafka3:9092`.
 
+Finally, you can configure Racecar directly in Ruby. The file `config/racecar.rb` will be automatically loaded if it exists; in it, you can configure Racecar using a simple API:
+
+```ruby
+Racecar.configure do |config|
+  # Each config variable can be set using a writer attribute.
+  config.brokers = ServiceDiscovery.find("kafka-brokers")
+end
+```
+
 #### Basic configuration
 
 * `brokers` – A list of Kafka brokers in the cluster that you're consuming from. Defaults to `localhost` on port 9092, the default Kafka port.

data/examples/producing_consumer.rb

@@ -0,0 +1,9 @@
+class ProducingConsumer < Racecar::Consumer
+  subscribes_to "messages", start_from_beginning: false
+
+  def process(message)
+    value = message.value.reverse
+
+    produce value, topic: "reverse-messages"
+  end
+end

data/lib/racecar/consumer.rb

@@ -23,6 +23,16 @@ module Racecar
       end
     end
 
+    def configure(producer:)
+      @_producer = producer
+    end
+
     def teardown; end
+
+    protected
+
+    def produce(value, topic:)
+      @_producer.produce(value, topic: topic)
+    end
   end
 end

data/lib/racecar/runner.rb

@@ -22,6 +22,7 @@ module Racecar
         connect_timeout: config.connect_timeout,
         socket_timeout: config.socket_timeout,
         ssl_ca_cert: config.ssl_ca_cert,
+        ssl_ca_cert_file_path: config.ssl_ca_cert_file_path,
         ssl_client_cert: config.ssl_client_cert,
         ssl_client_cert_key: config.ssl_client_cert_key,
         sasl_plain_username: config.sasl_plain_username,
@@ -52,6 +53,10 @@ module Racecar
         )
       end
 
+      # Configure the consumer with a producer so it can produce messages.
+      producer = kafka.producer
+      processor.configure(producer: producer)
+
       begin
         if processor.respond_to?(:process)
           consumer.each_message(max_wait_time: config.max_wait_time) do |message|
@@ -68,6 +73,7 @@ module Racecar
 
             @instrumenter.instrument("process_message.racecar", payload) do
               processor.process(message)
+              producer.deliver_messages
             end
           end
         elsif processor.respond_to?(:process_batch)
@@ -86,6 +92,7 @@ module Racecar
 
             @instrumenter.instrument("process_batch.racecar", payload) do
               processor.process_batch(batch)
+              producer.deliver_messages
             end
           end
         else

data/lib/racecar/version.rb

@@ -1,3 +1,3 @@
 module Racecar
-  VERSION = "0.3.5"
+  VERSION = "0.3.6"
 end

data/racecar.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "king_konf", "~> 0.1.8"
+  spec.add_runtime_dependency "king_konf", "~> 0.1.9"
   spec.add_runtime_dependency "ruby-kafka", "~> 0.4"
 
   spec.add_development_dependency "bundler", "~> 1.13"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.3.6
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-12-21 00:00:00.000000000 Z
+date: 2018-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.8
+        version: 0.1.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.8
+        version: 0.1.9
 - !ruby/object:Gem::Dependency
   name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement
@@ -103,6 +103,7 @@ files:
 - bin/setup
 - examples/batch_consumer.rb
 - examples/cat_consumer.rb
+- examples/producing_consumer.rb
 - exe/racecar
 - exe/racecarctl
 - lib/generators/racecar/consumer_generator.rb