racecar 0.5.0.beta1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5806641d4fcbd87d7625458de4ed60b19db3b4a8695d52dc062c72e1db281136
-  data.tar.gz: '08f9d6c439bcbc97bc05bcffb37d97b6eaf3b0a375deb85d1ad33dd2c004c9f7'
+  metadata.gz: 4be479ec5ba06da85a4e862849154a75adad8d6a41ca100f702f3dde4948bea8
+  data.tar.gz: 23824056ce3ca58258dd125daef3ffdae3a29f04e2f10987d2d20ef6e1a1a7fb
 SHA512:
-  metadata.gz: 18b8f848e4830f538b679cb3301f9bdcff36f0ccf4a4dfd0b7edc125aaa4a738164dd879d10366878ca3b74876506471809880d5ce23a607e22c8fcf0de1768d
-  data.tar.gz: 32af67f7bafd1320423e60f18895e2264c8fe623a1ff3ef96d457255c286a795dcee35ed83e1dd6b581baa14f9defc889757daedaf516623f46e7a7273d98a10
+  metadata.gz: 2ddba5fa6de0bb0002cf90a19923314f0f6003c0cc175160a98efaf53888160d47bf432cfad97bd49d261ac52730974e528edd612671d03ba4d8e1ecb52b14fc
+  data.tar.gz: 487fd52700aa79902ddc1a160cc2fdb38dabb2b2d4d3f3204d0b716ad5f1460554385d4f0abc3f35f60e38ad61855a828075db3901fc0abeb8cfcfec9be18550

data/.gitignore

@@ -7,4 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
-/vendor/bundle/
+/vendor/bundle/

data/CHANGELOG.md

@@ -2,9 +2,25 @@
 
 ## Unreleased
 
+## racecar v1.1.0
+
+* Require ruby-kafka v1.0 or higher.
+* Add error handling for required libraries (#149).
+
+## racecar v1.0.1
+
+* Add `--without-rails` option to boot consumer without Rails (#139).
+
+## racecar v1.0.0
+
+Unchanged from v0.5.0.
+
 ## racecar v0.5.0
 
 * Add support for manually sending heartbeats with `heartbeat` (#105).
+* Allow configuring `sasl_over_ssl`.
+* Add current directory to `$LOAD_PATH` only when `--require` option is used (#117).
+* Support for `ssl_verify_hostname` in the configuration (#120)
 
 ## racecar v0.4.2
 

data/README.md

@@ -205,8 +205,6 @@ The first time you execute `racecar` with a consumer class a _consumer group_ wi
 
 ### Producing messages
 
-**WARNING:** This is an alpha feature, and could cause weird and unpredictable errors. Use with caution.
-
 Consumers can produce messages themselves, allowing for powerful stream processing applications that transform and filter message streams. The API for this is simple:
 
 ```ruby
@@ -252,8 +250,8 @@ end
 
 * `brokers` – A list of Kafka brokers in the cluster that you're consuming from. Defaults to `localhost` on port 9092, the default Kafka port.
 * `client_id` – A string used to identify the client in logs and metrics.
-* `group_id` – The group id to use for a given group of consumers. Note that this _must_ be different for each consumer class. If left blank a group id is generated based on the consumer class name.
-* `group_id_prefix` – A prefix used when generating consumer group names. For instance, if you set the prefix to be `kevin.` and your consumer class is named `BaconConsumer`, the resulting consumer group will be named `kevin.bacon_consumer`.
+* `group_id` – The group id to use for a given group of consumers. Note that this _must_ be different for each consumer class. If left blank a group id is generated based on the consumer class name such that (for example) a consumer with the class name `BaconConsumer` would default to a group id of `bacon-consumer`.
+* `group_id_prefix` – A prefix used when generating consumer group names. For instance, if you set the prefix to be `kevin.` and your consumer class is named `BaconConsumer`, the resulting consumer group will be named `kevin.bacon-consumer`.
 
 #### Logging
 
@@ -334,6 +332,9 @@ Racecar supports configuring ruby-kafka's [Datadog](https://www.datadoghq.com/)
 * `datadog_namespace` – The namespace to use for Datadog metrics.
 * `datadog_tags` – Tags that should always be set on Datadog metrics.
 
+#### Consumers Without Rails ####
+
+By default, if Rails is detected, it will be automatically started when the consumer is started. There are cases where you might not want or need Rails. You can pass the `--without-rails` option when starting the consumer and Rails won't be started.
 
 ### Testing consumers
 
@@ -385,6 +386,44 @@ If you've ever used Heroku you'll recognize the format – indeed, deploying to
 
 With Foreman, you can easily run these processes locally by executing `foreman run`; in production you'll want to _export_ to another process management format such as Upstart or Runit. [capistrano-foreman](https://github.com/hyperoslo/capistrano-foreman) allows you to do this with Capistrano.
 
+#### Deploying to Kubernetes
+
+If you run your applications in Kubernetes, use the following [Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) spec as a starting point:
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: my-racecar-deployment
+  labels:
+    app: my-racecar
+spec:
+  replicas: 3 # <-- this will give us three consumers in the group.
+  selector:
+    matchLabels:
+      app: my-racecar
+  strategy:
+    type: Recreate # <-- this is the important part.
+  template:
+    metadata:
+      labels:
+        app: my-racecar
+    spec:
+      containers:
+      - name: my-racecar
+        image: my-racecar-image
+        command: ["bundle", "exec", "racecar", "MyConsumer"]
+        env: # <-- you can configure the consumer using environment variables!
+        - name: RACECAR_BROKERS
+          value: kafka1,kafka2,kafka3
+        - name: RACECAR_OFFSET_COMMIT_INTERVAL
+          value: 5
+```
+
+The important part is the `strategy.type` value, which tells Kubernetes how to upgrade from one version of your Deployment to another. Many services use so-called _rolling updates_, where some but not all containers are replaced with the new version. This is done so that, if the new version doesn't work, the old version is still there to serve most of the requests. For Kafka consumers, this doesn't work well. The reason is that every time a consumer joins or leaves a group, every other consumer in the group needs to stop and synchronize the list of partitions assigned to each group member. So if the group is updated in a rolling fashion, this synchronization would occur over and over again, causing undesirable double-processing of messages as consumers would start only to be synchronized shortly after.
+
+Instead, the `Recreate` update strategy should be used. It completely tears down the existing containers before starting all of the new containers simultaneously, allowing for a single synchronization stage and a much faster, more stable deployment update.
+
 
 #### Running consumers in the background
 

data/exe/racecar

@@ -3,8 +3,6 @@
 require "racecar"
 require "racecar/cli"
 
-$LOAD_PATH.unshift(Dir.pwd)
-
 begin
   Racecar::Cli.main(ARGV)
 rescue SignalException => e

data/lib/racecar.rb

@@ -22,6 +22,10 @@ module Racecar
     @config ||= Config.new
   end
 
+  def self.config=(config)
+    @config = config
+  end
+
   def self.configure
     yield config
   end

data/lib/racecar/cli.rb

@@ -23,10 +23,10 @@ module Racecar
     def run
       $stderr.puts "=> Starting Racecar consumer #{consumer_name}..."
 
-      RailsConfigFileLoader.load!
+      RailsConfigFileLoader.load! unless config.without_rails?
 
       if File.exist?("config/racecar.rb")
-        require "config/racecar"
+        require "./config/racecar"
       end
 
       # Find the consumer class by name.
@@ -94,11 +94,20 @@ module Racecar
     end
 
     def build_parser
+      load_path_modified = false
+
       OptionParser.new do |opts|
         opts.banner = "Usage: racecar MyConsumer [options]"
 
         opts.on("-r", "--require STRING", "Require a library before starting the consumer") do |lib|
-          require lib
+          $LOAD_PATH.unshift(Dir.pwd) unless load_path_modified
+          load_path_modified = true
+          begin
+            require lib
+          rescue => e
+            $stderr.puts "=> #{lib} failed to load: #{e.message}"
+            exit
+          end
         end
 
         opts.on("-l", "--log STRING", "Log to the specified file") do |logfile|
@@ -143,10 +152,10 @@ module Racecar
       require "kafka/datadog"
 
       datadog = Kafka::Datadog
-      datadog.host = config.datadog_host if config.datadog_host.present?
-      datadog.port = config.datadog_port if config.datadog_port.present?
-      datadog.namespace = config.datadog_namespace if config.datadog_namespace.present?
-      datadog.tags = config.datadog_tags if config.datadog_tags.present?
+      datadog.host = config.datadog_host unless config.datadog_host.nil?
+      datadog.port = config.datadog_port unless config.datadog_port.nil?
+      datadog.namespace = config.datadog_namespace unless config.datadog_namespace.nil?
+      datadog.tags = config.datadog_tags unless config.datadog_tags.nil?
     end
   end
 end

data/lib/racecar/config.rb

@@ -100,6 +100,9 @@ module Racecar
     desc "The SCRAM mechanism to use, either `sha256` or `sha512`"
     string :sasl_scram_mechanism, allowed_values: ["sha256", "sha512"]
 
+    desc "Whether to use SASL over SSL."
+    boolean :sasl_over_ssl, default: true
+
     desc "The file in which to store the Racecar process' PID when daemonized"
     string :pidfile
 
@@ -124,6 +127,12 @@ module Racecar
     desc "Tags that should always be set on Datadog metrics"
     list :datadog_tags
 
+    desc "Whether to check the server certificate is valid for the hostname"
+    boolean :ssl_verify_hostname, default: true
+
+    desc "Whether to boot Rails when starting the consumer"
+    boolean :without_rails, default: false
+
     # The error handler must be set directly on the object.
     attr_reader :error_handler
 

data/lib/racecar/runner.rb

@@ -32,7 +32,9 @@ module Racecar
         sasl_scram_username: config.sasl_scram_username,
         sasl_scram_password: config.sasl_scram_password,
         sasl_scram_mechanism: config.sasl_scram_mechanism,
+        sasl_over_ssl: config.sasl_over_ssl,
         ssl_ca_certs_from_system: config.ssl_ca_certs_from_system,
+        ssl_verify_hostname: config.ssl_verify_hostname
       )
 
       @consumer = kafka.consumer(

data/lib/racecar/version.rb

@@ -1,3 +1,3 @@
 module Racecar
-  VERSION = "0.5.0.beta1"
+  VERSION = "1.1.0"
 end

data/racecar.gemspec

@@ -21,9 +21,9 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency "king_konf", "~> 0.3.7"
-  spec.add_runtime_dependency "ruby-kafka", "~> 0.6"
+  spec.add_runtime_dependency "ruby-kafka", "~> 1.0"
 
-  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "bundler", [">= 1.13", "< 3"]
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 0.5.0.beta1
+  version: 1.1.0
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-01-21 00:00:00.000000000 Z
+date: 2020-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -31,28 +31,34 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.13'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.13'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -135,9 +141,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.7.6