deimos-ruby 1.8.2.pre.beta2 → 1.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b773de54759babb229e8f3bc6b42931dab7695b7197d601ad4ae08015c0b401f
-  data.tar.gz: 4af1aa98ffdfc013c4cac02fa316371ad138ec0d807ed58426a7ad57d6c1e719
+  metadata.gz: 237cb6cd22b6a057003fedae1caeec81c999a3d9b90d837bb8b0a4add43ca4e2
+  data.tar.gz: fab51933cdcd5c5fd10b8c92bbc67854fe56f23cdbec41660579f18b9075e09c
 SHA512:
-  metadata.gz: 319d01a4c42c5efe72bd4959fb57548102c3f2b4305f3d94510747fd372d516ebad3e34ff74f9e6744641728efce9eb7a20f2d89c37d6bd90847253698a05238
-  data.tar.gz: 9e6999cca87ca448167a9e61a0bfa5914b36fd74c053f132bd02ab5200be840984546469affccfd07ce77d4d098a2096b13e42febd4ce7059ec882c436580cd8
+  metadata.gz: 193b9b593b4f92edecc6c239eb6106fbe40e0acd1453694508f3fb02525166c39e4f1eba71230362493cd1647ab19e98a63ecf55f5371b844e9d0c38ffa5b2ea
+  data.tar.gz: e7d42ec7ec2bc864735c9230a925685935678d2dd0f3918426926dc9f358d5eb810dd5b324bcacaf9201c78bdfbbfa58037a4cac4cf4ca66257737440b3d2c97

data/CHANGELOG.md

@@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.8.2 - 2020-09-25
+
+### Features :star:
+- Add "disabled" config field to consumers to allow disabling
+  individual consumers without having to comment out their
+  entries and possibly affecting unit tests.
+
+### Fixes :wrench:
+- Prepend topic_prefix while encoding messages
+  (fixes [#37](https://github.com/flipp-oss/deimos/issues/37))
+- Raise error if producing without a topic
+  (fixes [#50](https://github.com/flipp-oss/deimos/issues/50))
+- Don't try to load producers/consumers when running rake tasks involving webpacker or assets
+
 ## 1.8.2-beta2 - 2020-09-15
 
 ### Features :star:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.8.2.pre.beta2)
+    deimos-ruby (1.8.2)
       avro_turf (~> 0.11)
       phobos (~> 1.9)
       ruby-kafka (~> 0.7)

data/docs/CONFIGURATION.md

@@ -79,6 +79,7 @@ topic|nil|Topic to produce to.
 schema|nil|This is optional but strongly recommended for testing purposes; this will validate against a local schema file used as the reader schema, as well as being able to write tests against this schema. This is recommended since it ensures you are always getting the values  you expect.
 namespace|nil|Namespace of the schema to use when finding it locally.
 key_config|nil|Configuration hash for message keys. See [Kafka Message Keys](../README.md#installation)
+disabled|false|Set to true to skip starting an actual listener for this consumer on startup.
 group_id|nil|ID of the consumer group.
 max_concurrency|1|Number of threads created for this listener. Each thread will behave as an independent consumer. They don't share any state.
 start_from_beginning|true|Once the consumer group has checkpointed its progress in the topic's partitions, the consumers will always start from the checkpointed offsets, regardless of config. As such, this setting only applies when the consumer initially starts consuming from a topic

data/lib/deimos/config/configurable.rb

@@ -245,6 +245,13 @@ module Deimos
 
       # Configure the settings with values.
       def configure(&block)
+        if defined?(Rake) && defined?(Rake.application)
+          tasks = Rake.application.top_level_tasks
+          if tasks.any? { |t| %w(assets webpacker yarn).include?(t.split(':').first) }
+            puts 'Skipping Deimos configuration since we are in JS/CSS compilation'
+            return
+          end
+        end
         config.run_callbacks(:configure) do
           config.instance_eval(&block)
         end

data/lib/deimos/config/configuration.rb

@@ -319,6 +319,10 @@ module Deimos
       # Key configuration (see docs).
       # @return [Hash]
       setting :key_config
+      # Set to true to ignore the consumer in the Phobos config and not actually start up a
+      # listener.
+      # @return [Boolean]
+      setting :disabled, false
 
       # These are the phobos "listener" configs. See CONFIGURATION.md for more
       # info.

data/lib/deimos/config/phobos_config.rb

@@ -63,8 +63,10 @@ module Deimos
       }
 
       p_config[:listeners] = self.consumer_objects.map do |consumer|
+        next nil if consumer.disabled
+
         hash = consumer.to_h.reject do |k, _|
-          %i(class_name schema namespace key_config backoff).include?(k)
+          %i(class_name schema namespace key_config backoff disabled).include?(k)
         end
         hash = hash.map { |k, v| [k, v.is_a?(Symbol) ? v.to_s : v] }.to_h
         hash[:handler] = consumer.class_name
@@ -73,6 +75,7 @@ module Deimos
         end
         hash
       end
+      p_config[:listeners].compact!
 
       if self.kafka.ssl.enabled
         %w(ca_cert client_cert client_cert_key).each do |key|

data/lib/deimos/producer.rb

@@ -104,6 +104,8 @@ module Deimos
                   Deimos.config.producers.disabled ||
                   Deimos.producers_disabled?(self)
 
+        raise 'Topic not specified. Please specify the topic.' if topic.blank?
+
         backend_class = determine_backend_class(sync, force_send)
         Deimos.instrument(
           'encode_messages',
@@ -183,7 +185,7 @@ module Deimos
                                     nil
                                   else
                                     encoder.encode(message.payload,
-                                                   topic: "#{config[:topic]}-value")
+                                                   topic: "#{Deimos.config.producers.topic_prefix}#{config[:topic]}-value")
                                   end
       end
 
@@ -201,9 +203,9 @@ module Deimos
         end
 
         if config[:key_field]
-          encoder.encode_key(config[:key_field], key, topic: "#{config[:topic]}-key")
+          encoder.encode_key(config[:key_field], key, topic: "#{Deimos.config.producers.topic_prefix}#{config[:topic]}-key")
         elsif config[:key_schema]
-          key_encoder.encode(key, topic: "#{config[:topic]}-key")
+          key_encoder.encode(key, topic: "#{Deimos.config.producers.topic_prefix}#{config[:topic]}-key")
         else
           key
         end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.8.2-beta2'
+  VERSION = '1.8.2'
 end

data/spec/config/configuration_spec.rb

@@ -6,6 +6,14 @@ class MyConfigConsumer < Deimos::Consumer
   def consume
   end
 end
+
+# Mock consumer 2
+class MyConfigConsumer2 < Deimos::Consumer
+  # :no-doc:
+  def consume
+  end
+end
+
 describe Deimos, 'configuration' do
   it 'should configure with deprecated fields' do
     logger = Logger.new(nil)
@@ -171,6 +179,13 @@ describe Deimos, 'configuration' do
         offset_retention_time 13
         heartbeat_interval 13
       end
+      consumer do
+        disabled true
+        class_name 'MyConfigConsumer2'
+        schema 'blah2'
+        topic 'blah2'
+        group_id 'myconsumerid2'
+      end
     end
 
     expect(described_class.config.phobos_config).

data/spec/producer_spec.rb

@@ -64,6 +64,14 @@ module ProducerTest
       end
       stub_const('MyErrorProducer', producer_class)
 
+      producer_class = Class.new(Deimos::Producer) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        topic nil
+        key_config none: true
+      end
+      stub_const('MyNoTopicProducer', producer_class)
+
     end
 
     it 'should fail on invalid message with error handler' do
@@ -228,6 +236,7 @@ module ProducerTest
     end
 
     it 'should encode the key' do
+      Deimos.configure { |c| c.producers.topic_prefix = nil }
       expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
       expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'my-topic-key')
       expect(MyProducer.encoder).to receive(:encode).with({
@@ -245,6 +254,21 @@ module ProducerTest
       )
     end
 
+    it 'should encode the key with topic prefix' do
+      Deimos.configure { |c| c.producers.topic_prefix = 'prefix.' }
+      expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'prefix.my-topic-key')
+      expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'prefix.my-topic-key')
+      expect(MyProducer.encoder).to receive(:encode).with({  'test_id' => 'foo',
+                                                             'some_int' => 123 },
+                                                          { topic: 'prefix.my-topic-value' })
+      expect(MyProducer.encoder).to receive(:encode).with({  'test_id' => 'bar',
+                                                             'some_int' => 124 },
+                                                          { topic: 'prefix.my-topic-value' })
+
+      MyProducer.publish_list([{ 'test_id' => 'foo', 'some_int' => 123 },
+                               { 'test_id' => 'bar', 'some_int' => 124 }])
+    end
+
     it 'should not encode with plaintext key' do
       expect(MyNonEncodedProducer.key_encoder).not_to receive(:encode_key)
 
@@ -296,6 +320,31 @@ module ProducerTest
       )
     end
 
+    it 'should raise error if blank topic is passed in explicitly' do
+      expect {
+        MyProducer.publish_list(
+          [{  'test_id' => 'foo',
+              'some_int' => 123 },
+           {  'test_id' => 'bar',
+              'some_int' => 124 }],
+          topic: ''
+        )
+      }.to raise_error(RuntimeError,
+                       'Topic not specified. Please specify the topic.')
+    end
+
+    it 'should raise error if the producer has not been initialized with a topic' do
+      expect {
+        MyNoTopicProducer.publish_list(
+          [{  'test_id' => 'foo',
+              'some_int' => 123 },
+           {  'test_id' => 'bar',
+              'some_int' => 124 }]
+        )
+      }.to raise_error(RuntimeError,
+                       'Topic not specified. Please specify the topic.')
+    end
+
     it 'should error with nothing set' do
       expect {
         MyErrorProducer.publish_list(

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.8.2.pre.beta2
+  version: 1.8.2
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-15 00:00:00.000000000 Z
+date: 2020-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -486,11 +486,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.3
 signing_key:
 specification_version: 4
 summary: Kafka libraries for Ruby.