deimos-ruby 1.8.2.pre.beta1 → 1.8.5

data/docs/PULL_REQUEST_TEMPLATE.md

@@ -28,6 +28,7 @@ Please describe the tests that you ran to verify your changes. Provide instructi
 - [ ] I have performed a self-review of my own code
 - [ ] I have commented my code, particularly in hard-to-understand areas
 - [ ] I have made corresponding changes to the documentation
+- [ ] I have added a line in the CHANGELOG describing this change, under the UNRELEASED heading
 - [ ] My changes generate no new warnings
 - [ ] I have added tests that prove my fix is effective or that my feature works
 - [ ] New and existing unit tests pass locally with my changes

data/docs/UPGRADING.md

@@ -0,0 +1,128 @@
+# Upgrading Deimos
+
+## Upgrading from < 1.5.0 to >= 1.5.0
+
+If you are using Confluent's schema registry to Avro-encode your
+messages, you will need to manually include the `avro_turf` gem
+in your Gemfile now.
+
+This update changes how to interact with Deimos's schema classes.
+Although these are meant to be internal, they are still "public"
+and can be used by calling code.
+
+Before 1.5.0:
+
+```ruby
+encoder = Deimos::AvroDataEncoder.new(schema: 'MySchema',
+                                      namespace: 'com.my-namespace')
+encoder.encode(my_payload)
+
+decoder = Deimos::AvroDataDecoder.new(schema: 'MySchema',
+                                      namespace: 'com.my-namespace')
+decoder.decode(my_payload)
+```
+
+After 1.5.0:
+```ruby
+backend = Deimos.schema_backend(schema: 'MySchema', namespace: 'com.my-namespace')
+backend.encode(my_payload)
+backend.decode(my_payload)
+```
+
+The two classes are different and if you are using them to e.g.
+inspect Avro schema fields, please look at the source code for the following:
+* `Deimos::SchemaBackends::Base`
+* `Deimos::SchemaBackends::AvroBase`
+* `Deimos::SchemaBackends::AvroSchemaRegistry`
+
+Deprecated `Deimos::TestHelpers.sent_messages` in favor of
+`Deimos::Backends::Test.sent_messages`.
+
+## Upgrading from < 1.4.0 to >= 1.4.0 
+
+Previously, configuration was handled as follows:
+* Kafka configuration, including listeners, lived in `phobos.yml`
+* Additional Deimos configuration would live in an initializer, e.g. `kafka.rb`
+* Producer and consumer configuration lived in each individual producer and consumer
+
+As of 1.4.0, all configuration is centralized in one initializer
+file, using default configuration.
+
+Before 1.4.0:
+```yaml
+# config/phobos.yml
+logger:
+  file: log/phobos.log
+  level: debug
+  ruby_kafka:
+    level: debug
+
+kafka:
+  client_id: phobos
+  connect_timeout: 15
+  socket_timeout: 15
+
+producer:
+  ack_timeout: 5
+  required_acks: :all
+  ...
+
+listeners:
+  - handler: ConsumerTest::MyConsumer
+    topic: my_consume_topic
+    group_id: my_group_id
+  - handler: ConsumerTest::MyBatchConsumer
+    topic: my_batch_consume_topic
+    group_id: my_batch_group_id
+    delivery: inline_batch
+```
+
+```ruby
+# kafka.rb
+Deimos.configure do |config|
+  config.reraise_consumer_errors = true
+  config.logger = Rails.logger
+  ...
+end
+
+# my_consumer.rb
+class ConsumerTest::MyConsumer < Deimos::Producer
+  namespace 'com.my-namespace'
+  schema 'MySchema'
+  topic 'MyTopic'
+  key_config field: :id
+end
+```
+
+After 1.4.0:
+```ruby
+kafka.rb
+Deimos.configure do
+  logger Rails.logger
+  kafka do
+    client_id 'phobos'
+    connect_timeout 15
+    socket_timeout 15
+  end
+  producers.ack_timeout 5
+  producers.required_acks :all
+  ...
+  consumer do
+    class_name 'ConsumerTest::MyConsumer'
+    topic 'my_consume_topic'
+    group_id 'my_group_id' 
+    namespace 'com.my-namespace'
+    schema 'MySchema'
+    topic 'MyTopic'
+    key_config field: :id
+  end
+  ...
+end
+```
+
+Note that the old configuration way *will* work if you set
+`config.phobos_config_file = "config/phobos.yml"`. You will
+get a number of deprecation notices, however. You can also still
+set the topic, namespace, etc. on the producer/consumer class,
+but it's much more convenient to centralize these configs
+in one place to see what your app does.

data/lib/deimos/active_record_consume/message_consumption.rb

@@ -26,6 +26,15 @@ module Deimos
 
       # :nodoc:
       def consume(payload, metadata)
+        unless self.process_message?(payload)
+          Deimos.config.logger.debug(
+            message: 'Skipping processing of message',
+            payload: payload,
+            metadata: metadata
+          )
+          return
+        end
+
         key = metadata.with_indifferent_access[:key]
         klass = self.class.config[:record_class]
         record = fetch_record(klass, (payload || {}).with_indifferent_access, key)

data/lib/deimos/active_record_consumer.rb

@@ -55,5 +55,13 @@ module Deimos
     def record_attributes(payload, _key=nil)
       @converter.convert(payload)
     end
+
+    # Override this message to conditionally save records
+    # @param payload [Hash] The kafka message as a hash
+    # @return [Boolean] if true, record is created/update.
+    #   If false, record processing is skipped but message offset is still committed.
+    def process_message?(_payload)
+      true
+    end
   end
 end

data/lib/deimos/backends/db.rb

@@ -14,7 +14,7 @@ module Deimos
             message = Deimos::KafkaMessage.new(
               message: m.encoded_payload ? m.encoded_payload.to_s.b : nil,
               topic: m.topic,
-              partition_key: m.partition_key || m.key
+              partition_key: partition_key_for(m)
             )
             message.key = m.encoded_key.to_s.b unless producer_class.config[:no_keys]
             message
@@ -26,6 +26,15 @@ module Deimos
             by: records.size
           )
         end
+
+        # @param message [Deimos::Message]
+        # @return [String] the partition key to use for this message
+        def partition_key_for(message)
+          return message.partition_key if message.partition_key.present?
+          return message.key unless message.key.is_a?(Hash)
+
+          message.key.to_yaml
+        end
       end
     end
   end

data/lib/deimos/config/configurable.rb

@@ -15,6 +15,11 @@ module Deimos
   #       enabled true
   #       ca_cert_file 'my_file'
   #     end
+  #     config.kafka do
+  #       ssl do
+  #         enabled true
+  #       end
+  #     end
   #   end
   # - Allows for arrays of configurations:
   #   Deimos.configure do |config|
@@ -245,6 +250,13 @@ module Deimos
 
       # Configure the settings with values.
       def configure(&block)
+        if defined?(Rake) && defined?(Rake.application)
+          tasks = Rake.application.top_level_tasks
+          if tasks.any? { |t| %w(assets webpacker yarn).include?(t.split(':').first) }
+            puts 'Skipping Deimos configuration since we are in JS/CSS compilation'
+            return
+          end
+        end
         config.run_callbacks(:configure) do
           config.instance_eval(&block)
         end

data/lib/deimos/config/configuration.rb

@@ -319,6 +319,10 @@ module Deimos
       # Key configuration (see docs).
       # @return [Hash]
       setting :key_config
+      # Set to true to ignore the consumer in the Phobos config and not actually start up a
+      # listener.
+      # @return [Boolean]
+      setting :disabled, false
 
       # These are the phobos "listener" configs. See CONFIGURATION.md for more
       # info.

data/lib/deimos/config/phobos_config.rb

@@ -63,8 +63,10 @@ module Deimos
       }
 
       p_config[:listeners] = self.consumer_objects.map do |consumer|
+        next nil if consumer.disabled
+
         hash = consumer.to_h.reject do |k, _|
-          %i(class_name schema namespace key_config backoff).include?(k)
+          %i(class_name schema namespace key_config backoff disabled).include?(k)
         end
         hash = hash.map { |k, v| [k, v.is_a?(Symbol) ? v.to_s : v] }.to_h
         hash[:handler] = consumer.class_name
@@ -73,6 +75,7 @@ module Deimos
         end
         hash
       end
+      p_config[:listeners].compact!
 
       if self.kafka.ssl.enabled
         %w(ca_cert client_cert client_cert_key).each do |key|

data/lib/deimos/kafka_source.rb

@@ -88,8 +88,9 @@ module Deimos
                                                   array_of_attributes,
                                                   options={})
         results = super
-        return unless self.kafka_config[:import]
-        return if array_of_attributes.empty?
+        if !self.kafka_config[:import] || array_of_attributes.empty?
+          return results
+        end
 
         # This will contain an array of hashes, where each hash is the actual
         # attribute hash that created the object.

data/lib/deimos/kafka_topic_info.rb

@@ -85,11 +85,8 @@ module Deimos
                       locked_at: Time.zone.now,
                       error: true,
                       retries: record.retries + 1 }
-        if ActiveRecord::VERSION::MAJOR >= 4
-          record.update!(attr_hash)
-        else
-          record.update_attributes!(attr_hash)
-        end
+        record.attributes = attr_hash
+        record.save!
       end
 
       # Update the locked_at timestamp to indicate that the producer is still

data/lib/deimos/producer.rb

@@ -104,6 +104,8 @@ module Deimos
                   Deimos.config.producers.disabled ||
                   Deimos.producers_disabled?(self)
 
+        raise 'Topic not specified. Please specify the topic.' if topic.blank?
+
         backend_class = determine_backend_class(sync, force_send)
         Deimos.instrument(
           'encode_messages',
@@ -183,7 +185,7 @@ module Deimos
                                     nil
                                   else
                                     encoder.encode(message.payload,
-                                                   topic: "#{config[:topic]}-value")
+                                                   topic: "#{Deimos.config.producers.topic_prefix}#{config[:topic]}-value")
                                   end
       end
 
@@ -201,9 +203,9 @@ module Deimos
         end
 
         if config[:key_field]
-          encoder.encode_key(config[:key_field], key, topic: "#{config[:topic]}-key")
+          encoder.encode_key(config[:key_field], key, topic: "#{Deimos.config.producers.topic_prefix}#{config[:topic]}-key")
         elsif config[:key_schema]
-          key_encoder.encode(key, topic: "#{config[:topic]}-key")
+          key_encoder.encode(key, topic: "#{Deimos.config.producers.topic_prefix}#{config[:topic]}-key")
         else
           key
         end

data/lib/deimos/schema_backends/avro_schema_coercer.rb

@@ -44,9 +44,11 @@ module Deimos
 
       case field_type
       when :int, :long
-        if val.is_a?(Integer) ||
-           _is_integer_string?(val) ||
-           int_classes.any? { |klass| val.is_a?(klass) }
+        if %w(timestamp-millis timestamp-micros).include?(type.logical_type)
+          val
+        elsif val.is_a?(Integer) ||
+              _is_integer_string?(val) ||
+              int_classes.any? { |klass| val.is_a?(klass) }
           val.to_i
         else
           val # this will fail

data/lib/deimos/schema_backends/avro_schema_registry.rb

@@ -15,7 +15,7 @@ module Deimos
 
       # @override
       def encode_payload(payload, schema: nil, topic: nil)
-        avro_turf_messaging.encode(payload, schema_name: schema, subject: topic)
+        avro_turf_messaging.encode(payload, schema_name: schema, subject: topic || schema)
       end
 
     private

data/lib/deimos/utils/db_poller.rb

@@ -142,7 +142,8 @@ module Deimos
         last_id = record.public_send(id_method)
         last_updated_at = last_updated(record)
         @producer.send_events(batch)
-        @info.update_attributes!(last_sent: last_updated_at, last_sent_id: last_id)
+        @info.attributes = { last_sent: last_updated_at, last_sent_id: last_id }
+        @info.save!
       end
     end
   end

data/lib/deimos/utils/db_producer.rb

@@ -190,11 +190,14 @@ module Deimos
         end
       end
 
+      # Produce messages in batches, reducing the size 1/10 if the batch is too
+      # large. Does not retry batches of messages that have already been sent.
       # @param batch [Array<Hash>]
       def produce_messages(batch)
         batch_size = batch.size
+        current_index = 0
         begin
-          batch.in_groups_of(batch_size, false).each do |group|
+          batch[current_index..-1].in_groups_of(batch_size, false).each do |group|
             @logger.debug("Publishing #{group.size} messages to #{@current_topic}")
             producer.publish_list(group)
             Deimos.config.metrics&.increment(
@@ -202,6 +205,7 @@ module Deimos
               tags: %W(status:success topic:#{@current_topic}),
               by: group.size
             )
+            current_index += group.size
             @logger.info("Sent #{group.size} messages to #{@current_topic}")
           end
         rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge,

data/lib/deimos/utils/inline_consumer.rb

@@ -6,6 +6,7 @@ module Deimos
   module Utils
     # Listener that can seek to get the last X messages in a topic.
     class SeekListener < Phobos::Listener
+      MAX_SEEK_RETRIES = 3
       attr_accessor :num_messages
 
       # :nodoc:
@@ -13,16 +14,22 @@ module Deimos
         @num_messages ||= 10
         @consumer = create_kafka_consumer
         @consumer.subscribe(topic, @subscribe_opts)
+        attempt = 0
 
         begin
+          attempt += 1
           last_offset = @kafka_client.last_offset_for(topic, 0)
           offset = last_offset - num_messages
           if offset.positive?
             Deimos.config.logger.info("Seeking to #{offset}")
-            @consumer.seek(topic, 1, offset)
+            @consumer.seek(topic, 0, offset)
           end
         rescue StandardError => e
-          "Could not seek to offset: #{e.message}"
+          if attempt < MAX_SEEK_RETRIES
+            sleep(1.seconds * attempt)
+            retry
+          end
+          log_error("Could not seek to offset: #{e.message} after #{MAX_SEEK_RETRIES} retries", listener_metadata)
         end
 
         instrument('listener.start_handler', listener_metadata) do
@@ -50,7 +57,6 @@ module Deimos
 
       # :nodoc:
       def consume(payload, metadata)
-        puts "Got #{payload}"
         self.class.total_messages << {
           key: metadata[:key],
           payload: payload

data/lib/deimos/utils/schema_controller_mixin.rb

@@ -28,14 +28,18 @@ module Deimos
 
         # Indicate which schemas should be assigned to actions.
         # @param actions [Symbol]
+        # @param kwactions [String]
         # @param request [String]
         # @param response [String]
-        def schemas(*actions, request: nil, response: nil)
+        def schemas(*actions, request: nil, response: nil, **kwactions)
           actions.each do |action|
             request ||= action.to_s.titleize
             response ||= action.to_s.titleize
             schema_mapping[action.to_s] = { request: request, response: response }
           end
+          kwactions.each do |key, val|
+            schema_mapping[key.to_s] = { request: val, response: val }
+          end
         end
 
         # @return [Hash<Symbol, String>]

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.8.2-beta1'
+  VERSION = '1.8.5'
 end

data/spec/active_record_consumer_spec.rb

@@ -137,5 +137,18 @@ module ActiveRecordConsumerTest
       expect(Widget.find_by_test_id('id1').some_int).to eq(3)
       expect(Widget.find_by_test_id('id2').some_int).to eq(4)
     end
+
+    it 'should not create record of process_message returns false' do
+      MyConsumer.any_instance.stub(:process_message?).and_return(false)
+      expect(Widget.count).to eq(0)
+      test_consume_message(MyConsumer, {
+                             test_id: 'abc',
+                             some_int: 3,
+                             updated_at: 1.day.ago.to_i,
+                             some_datetime_int: Time.zone.now.to_i,
+                             timestamp: 2.minutes.ago.to_s
+                           }, { call_original: true, key: 5 })
+      expect(Widget.count).to eq(0)
+    end
   end
 end