racecar 0.1.4 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 92c4f52171961fdbf4dcf6db7ed265eec6a729dd
-  data.tar.gz: 8bb0fc336b839f91da037cc7046dc64fa699acdd
+  metadata.gz: 63f11732ede854b710bf164d64cd1815e1edf588
+  data.tar.gz: 2f4451111982eaa874334ae309c8e6063fbd6bea
 SHA512:
-  metadata.gz: 180c8bc2e058b7626b1f2d3f277d176c12b75b27191222ae40f80cb893142037132b8f469c5d693482be317e554453019b09c2ff941bbf5fc88665825dc9c2b4
-  data.tar.gz: d19f91361e5d5a59914c74fa89e4d574792dbeb0e6c27cc1ddc153bd463dc23498ed8b0d9d76e63d1b525058af0eaad16cc75aab58f3926c6a825bb3fb3ca1c8
+  metadata.gz: 2ccc72f3e1e5e76f4c16f20451a281cd7640c4a643a75dea4c4100e41e071433a2a3c297cc06bfcb8fdcf066a22b20e8856176d2a238bb79ca241b42bbd8fd1f
+  data.tar.gz: b9a03db77deaae0e50e17822931ce0e46b1f499ce80b9edfaee82bd334115cc226041f9a138bc87266ade1b29fbcf5f2e97d1b12fbefce432b8e236aaa0bf3c5

data/README.md

@@ -116,13 +116,48 @@ The possible configuration keys are:
 * `offset_commit_interval` (_optional_) – How often to save the consumer's position in Kafka.
 * `heartbeat_interval` (_optional_) – How often to send a heartbeat message to Kafka.
 * `pause_timeout` (_optional_) – How long to pause a partition for if the consumer raises an exception while processing a message.
-* `connect_timeout` (_optional_) – How long to wait when trying to connect to a Kafka broker.
-* `socket_timeout` (_optional_) – How long to wait when trying to communicate with a Kafka broker.
+* `connect_timeout` (_optional_) – How long to wait when trying to connect to a Kafka broker. Default is 10 seconds.
+* `socket_timeout` (_optional_) – How long to wait when trying to communicate with a Kafka broker. Default is 30 seconds.
+* `max_wait_time` (_optional_) – How long to allow the Kafka brokers to wait before returning messages. A higher number means larger batches, at the cost of higher latency. Default is 5 seconds.
 
 Note that many of these configuration keys correspond directly with similarly named concepts in [ruby-kafka](https://github.com/zendesk/ruby-kafka) for more details on low-level operations, read that project's documentation.
 
 It's also possible to configure Racecar using environment variables. For any given configuration key, there should be a corresponding environment variable with the prefix `RACECAR_`, in upper case. For instance, in order to configure the client id, set `RACECAR_CLIENT_ID=some-id` in the process in which the Racecar consumer is launched. You can set `brokers` by passing a comma-separated list, e.g. `RACECAR_BROKERS=kafka1:9092,kafka2:9092,kafka3:9092`.
 
+### Testing consumers
+
+Since consumers are merely classes that implement a simple interface, they're dead simple to test.
+
+Here's an example of testing a consumer class using [RSpec](http://rspec.info/) and Rails:
+
+```ruby
+# app/consumers/create_contacts_consumer.rb
+#
+# Creates a Contact whenever an email address is written to the
+# `email-addresses` topic.
+class CreateContactsConsumer < Racecar::Consumer
+  subscribes_to "email-addresses"
+  
+  def process(message)
+    email = message.value
+    
+    Contact.create!(email: email)
+  end
+end
+
+# spec/consumers/create_contacts_consumer_spec.rb
+describe CreateContactsConsumer do
+  it "creates a Contact for each email address in the topic" do
+    message = double("message", value: "john@example.com")
+    consumer = CreateContactsConsumer.new
+    
+    consumer.process(message)
+    
+    expect(Contact.where(email: "john@example.com")).to exist
+  end
+end
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/exe/racecar

@@ -3,4 +3,6 @@
 require "racecar"
 require "racecar/cli"
 
+$LOAD_PATH.unshift(Dir.pwd)
+
 Racecar::Cli.main(ARGV)

data/exe/racecarctl

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require "racecar"
+require "racecar/ctl"
+
+Racecar::Ctl.main(ARGV)

data/lib/racecar/cli.rb

@@ -1,4 +1,5 @@
 require "optparse"
+require "racecar/config_loader"
 
 module Racecar
   module Cli
@@ -20,31 +21,10 @@ module Racecar
       parser.parse!(args)
 
       consumer_name = args.first or raise Racecar::Error, "no consumer specified"
-      config_file = "config/racecar.yml"
 
       puts "=> Starting Racecar consumer #{consumer_name}..."
 
-      begin
-        require "rails"
-
-        puts "=> Detected Rails, booting application..."
-
-        require "./config/environment"
-
-        Racecar.config.load_file(config_file, Rails.env)
-
-        if Racecar.config.log_to_stdout
-          # Write to STDOUT as well as to the log file.
-          console = ActiveSupport::Logger.new($stdout)
-          console.formatter = Rails.logger.formatter
-          console.level = Rails.logger.level
-          Rails.logger.extend(ActiveSupport::Logger.broadcast(console))
-        end
-
-        Racecar.logger = Rails.logger
-      rescue LoadError
-        # Not a Rails application.
-      end
+      ConfigLoader.load!
 
       # Find the consumer class by name.
       consumer_class = Kernel.const_get(consumer_name)

data/lib/racecar/config.rb

@@ -45,10 +45,21 @@ module Racecar
       # Default is to not pause partitions on processing errors.
       pause_timeout: 0,
 
-      connect_timeout: nil,
-      socket_timeout: nil,
-      error_handler: proc {},
+      # Default is to allow at most 10 seconds when connecting to a broker.
+      connect_timeout: 10,
+
+      # Default is to allow at most 30 seconds when reading or writing to
+      # a broker socket.
+      socket_timeout: 30,
+
+      # Default is to allow the brokers up to 5 seconds before returning
+      # messages.
       max_wait_time: 5,
+
+      # Default is to do nothing on exceptions.
+      error_handler: proc {},
+
+      # Default is to only log to the logger.
       log_to_stdout: false,
     }
 
@@ -65,6 +76,14 @@ module Racecar
           raise ConfigError, "required configuration key `#{key}` not defined"
         end
       end
+
+      if socket_timeout <= max_wait_time
+        raise ConfigError, "`socket_timeout` must be longer than `max_wait_time`"
+      end
+
+      if connect_timeout <= max_wait_time
+        raise ConfigError, "`connect_timeout` must be longer than `max_wait_time`"
+      end
     end
 
     def load_file(path, environment)

data/lib/racecar/config_loader.rb

@@ -0,0 +1,29 @@
+module Racecar
+  module ConfigLoader
+    def self.load!
+      config_file = "config/racecar.yml"
+
+      begin
+        require "rails"
+
+        puts "=> Detected Rails, booting application..."
+
+        require "./config/environment"
+
+        Racecar.config.load_file(config_file, Rails.env)
+
+        if Racecar.config.log_to_stdout
+          # Write to STDOUT as well as to the log file.
+          console = ActiveSupport::Logger.new($stdout)
+          console.formatter = Rails.logger.formatter
+          console.level = Rails.logger.level
+          Rails.logger.extend(ActiveSupport::Logger.broadcast(console))
+        end
+
+        Racecar.logger = Rails.logger
+      rescue LoadError
+        # Not a Rails application.
+      end
+    end
+  end
+end

data/lib/racecar/consumer.rb

@@ -1,6 +1,6 @@
 module Racecar
   class Consumer
-    Subscription = Struct.new(:topic, :start_from_beginning)
+    Subscription = Struct.new(:topic, :start_from_beginning, :max_bytes_per_partition)
 
     class << self
       attr_accessor :max_wait_time
@@ -11,9 +11,14 @@ module Racecar
       end
 
       # Adds one or more topic subscriptions.
-      def subscribes_to(*topics, start_from_beginning: true)
+      #
+      # start_from_beginning    - whether to start from the beginning or the end of each
+      #                           partition.
+      # max_bytes_per_partition - the maximum number of bytes to fetch from each partition
+      #                           at a time.
+      def subscribes_to(*topics, start_from_beginning: true, max_bytes_per_partition: 1048576)
         topics.each do |topic|
-          subscriptions << Subscription.new(topic, start_from_beginning)
+          subscriptions << Subscription.new(topic, start_from_beginning, max_bytes_per_partition)
         end
       end
     end

data/lib/racecar/ctl.rb

@@ -0,0 +1,58 @@
+require "optparse"
+require "racecar/config_loader"
+
+module Racecar
+  class Ctl
+    ProduceMessage = Struct.new(:value, :key, :topic)
+
+    def self.main(args)
+      command = args.shift or raise Racecar::Error, "no command specified"
+
+      ctl = new
+
+      if ctl.respond_to?(command)
+        ctl.send(command, args)
+      else
+        raise Racecar::Error, "invalid command: #{command}"
+      end
+    end
+
+    def produce(args)
+      message = ProduceMessage.new
+
+      parser = OptionParser.new do |opts|
+        opts.banner = "Usage: racecarctl produce [options]"
+
+        opts.on("-v", "--value VALUE", "Set the message value") do |value|
+          message.value = value
+        end
+
+        opts.on("-k", "--key KEY", "Set the message key") do |key|
+          message.key = key
+        end
+
+        opts.on("-t", "--topic TOPIC", "Set the message topic") do |topic|
+          message.topic = topic
+        end
+      end
+
+      parser.parse!(args)
+
+      ConfigLoader.load!
+
+      Racecar.config.validate!
+
+      kafka = Kafka.new(
+        client_id: Racecar.config.client_id,
+        seed_brokers: Racecar.config.brokers,
+        logger: Racecar.logger,
+        connect_timeout: Racecar.config.connect_timeout,
+        socket_timeout: Racecar.config.socket_timeout,
+      )
+
+      kafka.deliver_message(message.value, key: message.key, topic: message.topic)
+
+      puts "=> Delivered message to Kafka cluster"
+    end
+  end
+end

data/lib/racecar/runner.rb

@@ -29,10 +29,11 @@ module Racecar
       trap("INT") { consumer.stop }
 
       config.subscriptions.each do |subscription|
-        topic = subscription.topic
-        start_from_beginning = subscription.start_from_beginning
-
-        consumer.subscribe(topic, start_from_beginning: start_from_beginning)
+        consumer.subscribe(
+          subscription.topic,
+          start_from_beginning: subscription.start_from_beginning,
+          max_bytes_per_partition: subscription.max_bytes_per_partition,
+        )
       end
 
       begin

data/lib/racecar/version.rb

@@ -1,3 +1,3 @@
 module Racecar
-  VERSION = "0.1.4"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-06-21 00:00:00.000000000 Z
+date: 2017-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-kafka
@@ -73,6 +73,7 @@ email:
 - bquorning@zendesk.com
 executables:
 - racecar
+- racecarctl
 extensions: []
 extra_rdoc_files: []
 files:
@@ -86,6 +87,7 @@ files:
 - bin/setup
 - examples/cat_consumer.rb
 - exe/racecar
+- exe/racecarctl
 - lib/generators/racecar/consumer_generator.rb
 - lib/generators/racecar/install_generator.rb
 - lib/generators/templates/consumer.rb.erb
@@ -93,7 +95,9 @@ files:
 - lib/racecar.rb
 - lib/racecar/cli.rb
 - lib/racecar/config.rb
+- lib/racecar/config_loader.rb
 - lib/racecar/consumer.rb
+- lib/racecar/ctl.rb
 - lib/racecar/env_loader.rb
 - lib/racecar/runner.rb
 - lib/racecar/version.rb