deimos-ruby 1.8.1.pre.beta9 → 1.8.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 18ef8f674c86fcdba3fc7a2556162b1104084fd38c6ee8046ed773e9352633bd
-  data.tar.gz: 60f5a05874a0cd2cd6abceb7cfc5af1a01068a2a3cc4cc4e65dd59d42783468a
+  metadata.gz: 497160ac15346d0acb471818c0f02e10f745761ba3777a70039b1a762e8f4edb
+  data.tar.gz: d85717deea2ac68d6c995d1dce1c42187d4fe5585b577c4468cafff7f0f71097
 SHA512:
-  metadata.gz: '0963acc646195fac41c3aaa71c67bbdc253bf26fa650c6132096a7912b665bb1956e3511f41db08309fbc9db35a6717c7a48f17a340039732bced4c5b6bd8b7a'
-  data.tar.gz: 0c642071d49e099d8516ed05c5856b1bba00a432c30567b3c79491654b0e1b1cc43e51a4c20f9765e0f2cbcd4af6550fbaa7620b1693d9ed342202609e496914
+  metadata.gz: 441577e56812dbc9d0dbbf22b43cb76e390c792165f7d21febcdf17a309fe85ec57bd9ddaad0929ddd3d49dd602a94601fc2660b124419e5943460e2fab94aa5
+  data.tar.gz: 2fdd3a94d91130c2430d189165504bdf437907fa1a9a459841bd4e260a2ff906469fb0f6989a8faf6c140cc133d22639af91a89549c85a9aeef2145636aa7b58

data/CHANGELOG.md

@@ -7,6 +7,56 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.8.4 - 2020-12-02
+
+### Features :star:
+- Add overridable "process_message?" method to ActiveRecordConsumer to allow for skipping of saving/updating records
+
+### Fixes :wrench:
+
+- Do not apply type coercion to `timestamp-millis` and `timestamp-micros` logical types (fixes [#97](https://github.com/flipp-oss/deimos/issues/97))
+
+## 1.8.3 - 2020-11-18
+
+### Fixes :wrench:
+- Do not resend already sent messages when splitting up batches
+  (fixes [#24](https://github.com/flipp-oss/deimos/issues/24))
+- KafkaSource crashing on bulk-imports if import hooks are disabled
+  (fixes [#73](https://github.com/flipp-oss/deimos/issues/73))
+- #96 Use string-safe encoding for partition keys
+- Retry on offset seek failures in inline consumer
+  (fixes [#5](Inline consumer should use retries when seeking))
+
+## 1.8.2 - 2020-09-25
+
+### Features :star:
+- Add "disabled" config field to consumers to allow disabling
+  individual consumers without having to comment out their
+  entries and possibly affecting unit tests.
+
+### Fixes :wrench:
+- Prepend topic_prefix while encoding messages
+  (fixes [#37](https://github.com/flipp-oss/deimos/issues/37))
+- Raise error if producing without a topic
+  (fixes [#50](https://github.com/flipp-oss/deimos/issues/50))
+- Don't try to load producers/consumers when running rake tasks involving webpacker or assets
+
+## 1.8.2-beta2 - 2020-09-15
+
+### Features :star:
+
+- Add details on using schema backend directly in README.
+- Default to the provided schema if topic is not provided when
+  encoding to `AvroSchemaRegistry`.
+- Add mapping syntax for the `schema` call in `SchemaControllerMixin`.
+
+## 1.8.2-beta1 - 2020-09-09
+
+### Features :star:
+
+- Added the ability to specify the topic for `publish`
+and `publish_list` in a producer
+
 ## 1.8.1-beta9 - 2020-08-27
 
 ### Fixes :wrench:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.8.1.pre.beta8)
+    deimos-ruby (1.8.4)
       avro_turf (~> 0.11)
       phobos (~> 1.9)
       ruby-kafka (~> 0.7)

data/README.md

@@ -11,6 +11,7 @@ a useful toolbox of goodies for Ruby-based Kafka development.
 Built on Phobos and hence Ruby-Kafka.
 
 <!--ts-->
+   * [Additional Documentation](#additional-documentation)
    * [Installation](#installation)
    * [Versioning](#versioning)
    * [Configuration](#configuration)
@@ -29,9 +30,20 @@ Built on Phobos and hence Ruby-Kafka.
    * [Metrics](#metrics)
    * [Testing](#testing)
         * [Integration Test Helpers](#integration-test-helpers)
+   * [Utilities](#utilities)
    * [Contributing](#contributing) 
 <!--te-->
 
+# Additional Documentation
+
+Please see the following for further information not covered by this readme:
+
+* [Architecture Design](docs/ARCHITECTURE.md)
+* [Configuration Reference](docs/CONFIGURATION.md)
+* [Database Backend Feature](docs/DATABASE_BACKEND.md)
+* [Upgrading Deimos](docs/UPGRADING.md)
+* [Contributing to Integration Tests](docs/INTEGRATION_TESTS.md)
+
 # Installation
 
 Add this line to your application's Gemfile:
@@ -108,6 +120,7 @@ class MyProducer < Deimos::Producer
         'some-key2' => an_object.bar
       }
       # You can also publish an array with self.publish_list(payloads)
+      # You may specify the topic here with self.publish(payload, topic: 'my-topic')
       self.publish(payload)
     end
     
@@ -481,6 +494,12 @@ class WhateverController < ApplicationController
   # will look for: my.namespace.requests.Index.avsc
   #                my.namespace.responses.Index.avsc 
   
+  # Can use mapping to change the schema but keep the namespaces,
+  # i.e. use the same schema name across the two namespaces
+  schemas create: 'CreateTopic'
+  # will look for: my.namespace.requests.CreateTopic.avsc
+  #                my.namespace.responses.CreateTopic.avsc 
+
   # If all routes use the default, you can add them all at once
   schemas :index, :show, :update
 
@@ -607,6 +626,15 @@ class MyConsumer < Deimos::ActiveRecordConsumer
   def record_key(payload)
     super
   end
+
+  # Optional override, returns true by default.
+  # When this method returns true, a record corresponding to the message
+  # is created/updated.
+  # When this method returns false, message processing is skipped and a
+  # corresponding record will NOT be created/updated.
+  def process_message?(payload)
+    super
+  end
 end
 ```
 
@@ -988,13 +1016,24 @@ Deimos::Utils::InlineConsumer.get_messages_for(
 )
 ```
 
+## Utilities
+
+You can use your configured schema backend directly if you want to
+encode and decode payloads outside of the context of sending messages.
+
+```ruby
+backend = Deimos.schema_backend(schema: 'MySchema', namespace: 'com.my-namespace')
+encoded = backend.encode(my_payload)
+decoded = backend.decode(my_encoded_payload)
+coerced = backend.coerce(my_payload) # coerce to correct types
+backend.validate(my_payload) # throws an error if not valid
+fields = backend.schema_fields # list of fields defined in the schema
+```
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/flipp-oss/deimos .
 
-We have more information on the [internal architecture](docs/ARCHITECTURE.md) of Deimos
-for contributors!
-
 ### Linting
 
 Deimos uses Rubocop to lint the code. Please run Rubocop on your code 

data/docs/CONFIGURATION.md

@@ -79,6 +79,7 @@ topic|nil|Topic to produce to.
 schema|nil|This is optional but strongly recommended for testing purposes; this will validate against a local schema file used as the reader schema, as well as being able to write tests against this schema. This is recommended since it ensures you are always getting the values  you expect.
 namespace|nil|Namespace of the schema to use when finding it locally.
 key_config|nil|Configuration hash for message keys. See [Kafka Message Keys](../README.md#installation)
+disabled|false|Set to true to skip starting an actual listener for this consumer on startup.
 group_id|nil|ID of the consumer group.
 max_concurrency|1|Number of threads created for this listener. Each thread will behave as an independent consumer. They don't share any state.
 start_from_beginning|true|Once the consumer group has checkpointed its progress in the topic's partitions, the consumers will always start from the checkpointed offsets, regardless of config. As such, this setting only applies when the consumer initially starts consuming from a topic

data/docs/INTEGRATION_TESTS.md

@@ -0,0 +1,52 @@
+# Running Integration Tests
+
+This repo includes integration tests in the [spec/utils](spec/utils) directory.
+Here, there are tests for more deimos features that include a database integration like
+* [Database Poller](README.md#Database Poller)
+* [Database Backend](docs/DATABASE_BACKEND.md)
+* [Deadlock Retrying](lib/deimos/utils/deadlock_retry.rb)
+
+You will need to set up the following databases to develop and create unit tests in these test suites.
+* [SQLite](#SQLite)
+* [MySQL](#MySQL)
+* [PostgreSQL](#PostgreSQL)
+
+### SQLite
+This database is covered through the `sqlite3` gem. 
+
+## MySQL
+### Setting up a local MySQL server (Mac)
+```bash
+# Download MySQL (Optionally, choose a version you are comfortable with)
+brew install mysql
+# Start automatically after rebooting your machine
+brew services start mysql
+
+# Cleanup once you are done with MySQL
+brew services stop mysql
+```
+
+## PostgreSQL
+### Setting up a local PostgreSQL server (Mac)
+```bash
+# Install postgres if it's not already installed
+brew install postgres
+
+# Initialize and Start up postgres db
+brew services start postgres
+initdb /usr/local/var/postgres
+# Create the default database and user
+# Use the password "root"
+createuser -s --password postgres
+
+# Cleanup once done with Postgres
+killall postgres
+brew services stop postgres
+```
+
+## Running Integration Tests
+You must specify the tag "integration" when running these these test suites.
+This can be done through the CLI with the `--tag integration` argument.
+```bash
+rspec spec/utils/ --tag integration
+```

data/docs/PULL_REQUEST_TEMPLATE.md

@@ -28,6 +28,7 @@ Please describe the tests that you ran to verify your changes. Provide instructi
 - [ ] I have performed a self-review of my own code
 - [ ] I have commented my code, particularly in hard-to-understand areas
 - [ ] I have made corresponding changes to the documentation
+- [ ] I have added a line in the CHANGELOG describing this change, under the UNRELEASED heading
 - [ ] My changes generate no new warnings
 - [ ] I have added tests that prove my fix is effective or that my feature works
 - [ ] New and existing unit tests pass locally with my changes

data/docs/UPGRADING.md

@@ -0,0 +1,128 @@
+# Upgrading Deimos
+
+## Upgrading from < 1.5.0 to >= 1.5.0
+
+If you are using Confluent's schema registry to Avro-encode your
+messages, you will need to manually include the `avro_turf` gem
+in your Gemfile now.
+
+This update changes how to interact with Deimos's schema classes.
+Although these are meant to be internal, they are still "public"
+and can be used by calling code.
+
+Before 1.5.0:
+
+```ruby
+encoder = Deimos::AvroDataEncoder.new(schema: 'MySchema',
+                                      namespace: 'com.my-namespace')
+encoder.encode(my_payload)
+
+decoder = Deimos::AvroDataDecoder.new(schema: 'MySchema',
+                                      namespace: 'com.my-namespace')
+decoder.decode(my_payload)
+```
+
+After 1.5.0:
+```ruby
+backend = Deimos.schema_backend(schema: 'MySchema', namespace: 'com.my-namespace')
+backend.encode(my_payload)
+backend.decode(my_payload)
+```
+
+The two classes are different and if you are using them to e.g.
+inspect Avro schema fields, please look at the source code for the following:
+* `Deimos::SchemaBackends::Base`
+* `Deimos::SchemaBackends::AvroBase`
+* `Deimos::SchemaBackends::AvroSchemaRegistry`
+
+Deprecated `Deimos::TestHelpers.sent_messages` in favor of
+`Deimos::Backends::Test.sent_messages`.
+
+## Upgrading from < 1.4.0 to >= 1.4.0 
+
+Previously, configuration was handled as follows:
+* Kafka configuration, including listeners, lived in `phobos.yml`
+* Additional Deimos configuration would live in an initializer, e.g. `kafka.rb`
+* Producer and consumer configuration lived in each individual producer and consumer
+
+As of 1.4.0, all configuration is centralized in one initializer
+file, using default configuration.
+
+Before 1.4.0:
+```yaml
+# config/phobos.yml
+logger:
+  file: log/phobos.log
+  level: debug
+  ruby_kafka:
+    level: debug
+
+kafka:
+  client_id: phobos
+  connect_timeout: 15
+  socket_timeout: 15
+
+producer:
+  ack_timeout: 5
+  required_acks: :all
+  ...
+
+listeners:
+  - handler: ConsumerTest::MyConsumer
+    topic: my_consume_topic
+    group_id: my_group_id
+  - handler: ConsumerTest::MyBatchConsumer
+    topic: my_batch_consume_topic
+    group_id: my_batch_group_id
+    delivery: inline_batch
+```
+
+```ruby
+# kafka.rb
+Deimos.configure do |config|
+  config.reraise_consumer_errors = true
+  config.logger = Rails.logger
+  ...
+end
+
+# my_consumer.rb
+class ConsumerTest::MyConsumer < Deimos::Producer
+  namespace 'com.my-namespace'
+  schema 'MySchema'
+  topic 'MyTopic'
+  key_config field: :id
+end
+```
+
+After 1.4.0:
+```ruby
+kafka.rb
+Deimos.configure do
+  logger Rails.logger
+  kafka do
+    client_id 'phobos'
+    connect_timeout 15
+    socket_timeout 15
+  end
+  producers.ack_timeout 5
+  producers.required_acks :all
+  ...
+  consumer do
+    class_name 'ConsumerTest::MyConsumer'
+    topic 'my_consume_topic'
+    group_id 'my_group_id' 
+    namespace 'com.my-namespace'
+    schema 'MySchema'
+    topic 'MyTopic'
+    key_config field: :id
+  end
+  ...
+end
+```
+
+Note that the old configuration way *will* work if you set
+`config.phobos_config_file = "config/phobos.yml"`. You will
+get a number of deprecation notices, however. You can also still
+set the topic, namespace, etc. on the producer/consumer class,
+but it's much more convenient to centralize these configs
+in one place to see what your app does.

data/lib/deimos/active_record_consume/message_consumption.rb

@@ -26,6 +26,15 @@ module Deimos
 
       # :nodoc:
       def consume(payload, metadata)
+        unless self.process_message?(payload)
+          Deimos.config.logger.debug(
+            message: 'Skipping processing of message',
+            payload: payload,
+            metadata: metadata
+          )
+          return
+        end
+
         key = metadata.with_indifferent_access[:key]
         klass = self.class.config[:record_class]
         record = fetch_record(klass, (payload || {}).with_indifferent_access, key)

data/lib/deimos/active_record_consumer.rb

@@ -55,5 +55,13 @@ module Deimos
     def record_attributes(payload, _key=nil)
       @converter.convert(payload)
     end
+
+    # Override this message to conditionally save records
+    # @param payload [Hash] The kafka message as a hash
+    # @return [Boolean] if true, record is created/update.
+    #   If false, record processing is skipped but message offset is still committed.
+    def process_message?(_payload)
+      true
+    end
   end
 end

data/lib/deimos/backends/db.rb

@@ -14,7 +14,7 @@ module Deimos
             message = Deimos::KafkaMessage.new(
               message: m.encoded_payload ? m.encoded_payload.to_s.b : nil,
               topic: m.topic,
-              partition_key: m.partition_key || m.key
+              partition_key: partition_key_for(m)
             )
             message.key = m.encoded_key.to_s.b unless producer_class.config[:no_keys]
             message
@@ -26,6 +26,15 @@ module Deimos
             by: records.size
           )
         end
+
+        # @param message [Deimos::Message]
+        # @return [String] the partition key to use for this message
+        def partition_key_for(message)
+          return message.partition_key if message.partition_key.present?
+          return message.key unless message.key.is_a?(Hash)
+
+          message.key.to_yaml
+        end
       end
     end
   end

data/lib/deimos/config/configurable.rb

@@ -15,6 +15,11 @@ module Deimos
   #       enabled true
   #       ca_cert_file 'my_file'
   #     end
+  #     config.kafka do
+  #       ssl do
+  #         enabled true
+  #       end
+  #     end
   #   end
   # - Allows for arrays of configurations:
   #   Deimos.configure do |config|
@@ -245,6 +250,13 @@ module Deimos
 
       # Configure the settings with values.
       def configure(&block)
+        if defined?(Rake) && defined?(Rake.application)
+          tasks = Rake.application.top_level_tasks
+          if tasks.any? { |t| %w(assets webpacker yarn).include?(t.split(':').first) }
+            puts 'Skipping Deimos configuration since we are in JS/CSS compilation'
+            return
+          end
+        end
         config.run_callbacks(:configure) do
           config.instance_eval(&block)
         end

data/lib/deimos/config/configuration.rb

@@ -319,6 +319,10 @@ module Deimos
       # Key configuration (see docs).
       # @return [Hash]
       setting :key_config
+      # Set to true to ignore the consumer in the Phobos config and not actually start up a
+      # listener.
+      # @return [Boolean]
+      setting :disabled, false
 
       # These are the phobos "listener" configs. See CONFIGURATION.md for more
       # info.

data/lib/deimos/config/phobos_config.rb

@@ -63,8 +63,10 @@ module Deimos
       }
 
       p_config[:listeners] = self.consumer_objects.map do |consumer|
+        next nil if consumer.disabled
+
         hash = consumer.to_h.reject do |k, _|
-          %i(class_name schema namespace key_config backoff).include?(k)
+          %i(class_name schema namespace key_config backoff disabled).include?(k)
         end
         hash = hash.map { |k, v| [k, v.is_a?(Symbol) ? v.to_s : v] }.to_h
         hash[:handler] = consumer.class_name
@@ -73,6 +75,7 @@ module Deimos
         end
         hash
       end
+      p_config[:listeners].compact!
 
       if self.kafka.ssl.enabled
         %w(ca_cert client_cert client_cert_key).each do |key|

data/lib/deimos/kafka_source.rb

@@ -88,8 +88,9 @@ module Deimos
                                                   array_of_attributes,
                                                   options={})
         results = super
-        return unless self.kafka_config[:import]
-        return if array_of_attributes.empty?
+        if !self.kafka_config[:import] || array_of_attributes.empty?
+          return results
+        end
 
         # This will contain an array of hashes, where each hash is the actual
         # attribute hash that created the object.

data/lib/deimos/producer.rb

@@ -87,8 +87,9 @@ module Deimos
 
       # Publish the payload to the topic.
       # @param payload [Hash] with an optional payload_key hash key.
-      def publish(payload)
-        publish_list([payload])
+      # @param topic [String] if specifying the topic
+      def publish(payload, topic: self.topic)
+        publish_list([payload], topic: topic)
       end
 
       # Publish a list of messages.
@@ -97,11 +98,14 @@ module Deimos
       # whether to publish synchronously.
       # @param force_send [Boolean] if true, ignore the configured backend
       # and send immediately to Kafka.
-      def publish_list(payloads, sync: nil, force_send: false)
+      # @param topic [String] if specifying the topic
+      def publish_list(payloads, sync: nil, force_send: false, topic: self.topic)
         return if Deimos.config.kafka.seed_brokers.blank? ||
                   Deimos.config.producers.disabled ||
                   Deimos.producers_disabled?(self)
 
+        raise 'Topic not specified. Please specify the topic.' if topic.blank?
+
         backend_class = determine_backend_class(sync, force_send)
         Deimos.instrument(
           'encode_messages',
@@ -110,7 +114,7 @@ module Deimos
           payloads: payloads
         ) do
           messages = Array(payloads).map { |p| Deimos::Message.new(p, self) }
-          messages.each(&method(:_process_message))
+          messages.each { |m| _process_message(m, topic) }
           messages.in_groups_of(MAX_BATCH_SIZE, false) do |batch|
             self.produce_batch(backend_class, batch)
           end
@@ -163,7 +167,8 @@ module Deimos
     private
 
       # @param message [Message]
-      def _process_message(message)
+      # @param topic [String]
+      def _process_message(message, topic)
         # this violates the Law of Demeter but it has to happen in a very
         # specific order and requires a bunch of methods on the producer
         # to work correctly.
@@ -175,12 +180,12 @@ module Deimos
         message.payload = nil if message.payload.blank?
         message.coerce_fields(encoder)
         message.encoded_key = _encode_key(message.key)
-        message.topic = self.topic
+        message.topic = topic
         message.encoded_payload = if message.payload.nil?
                                     nil
                                   else
                                     encoder.encode(message.payload,
-                                                   topic: "#{config[:topic]}-value")
+                                                   topic: "#{Deimos.config.producers.topic_prefix}#{config[:topic]}-value")
                                   end
       end
 
@@ -198,9 +203,9 @@ module Deimos
         end
 
         if config[:key_field]
-          encoder.encode_key(config[:key_field], key, topic: "#{config[:topic]}-key")
+          encoder.encode_key(config[:key_field], key, topic: "#{Deimos.config.producers.topic_prefix}#{config[:topic]}-key")
         elsif config[:key_schema]
-          key_encoder.encode(key, topic: "#{config[:topic]}-key")
+          key_encoder.encode(key, topic: "#{Deimos.config.producers.topic_prefix}#{config[:topic]}-key")
         else
           key
         end

data/lib/deimos/schema_backends/avro_schema_coercer.rb

@@ -44,9 +44,11 @@ module Deimos
 
       case field_type
       when :int, :long
-        if val.is_a?(Integer) ||
-           _is_integer_string?(val) ||
-           int_classes.any? { |klass| val.is_a?(klass) }
+        if %w(timestamp-millis timestamp-micros).include?(type.logical_type)
+          val
+        elsif val.is_a?(Integer) ||
+              _is_integer_string?(val) ||
+              int_classes.any? { |klass| val.is_a?(klass) }
           val.to_i
         else
           val # this will fail

data/lib/deimos/schema_backends/avro_schema_registry.rb

@@ -15,7 +15,7 @@ module Deimos
 
       # @override
       def encode_payload(payload, schema: nil, topic: nil)
-        avro_turf_messaging.encode(payload, schema_name: schema, subject: topic)
+        avro_turf_messaging.encode(payload, schema_name: schema, subject: topic || schema)
       end
 
     private

data/lib/deimos/utils/db_producer.rb

@@ -190,11 +190,14 @@ module Deimos
         end
       end
 
+      # Produce messages in batches, reducing the size 1/10 if the batch is too
+      # large. Does not retry batches of messages that have already been sent.
       # @param batch [Array<Hash>]
       def produce_messages(batch)
         batch_size = batch.size
+        current_index = 0
         begin
-          batch.in_groups_of(batch_size, false).each do |group|
+          batch[current_index..-1].in_groups_of(batch_size, false).each do |group|
             @logger.debug("Publishing #{group.size} messages to #{@current_topic}")
             producer.publish_list(group)
             Deimos.config.metrics&.increment(
@@ -202,6 +205,7 @@ module Deimos
               tags: %W(status:success topic:#{@current_topic}),
               by: group.size
             )
+            current_index += group.size
             @logger.info("Sent #{group.size} messages to #{@current_topic}")
           end
         rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge,

data/lib/deimos/utils/inline_consumer.rb

@@ -6,6 +6,7 @@ module Deimos
   module Utils
     # Listener that can seek to get the last X messages in a topic.
     class SeekListener < Phobos::Listener
+      MAX_SEEK_RETRIES = 3
       attr_accessor :num_messages
 
       # :nodoc:
@@ -13,8 +14,10 @@ module Deimos
         @num_messages ||= 10
         @consumer = create_kafka_consumer
         @consumer.subscribe(topic, @subscribe_opts)
+        attempt = 0
 
         begin
+          attempt += 1
           last_offset = @kafka_client.last_offset_for(topic, 0)
           offset = last_offset - num_messages
           if offset.positive?
@@ -22,7 +25,11 @@ module Deimos
             @consumer.seek(topic, 0, offset)
           end
         rescue StandardError => e
-          "Could not seek to offset: #{e.message}"
+          if attempt < MAX_SEEK_RETRIES
+            sleep(1.seconds * attempt)
+            retry
+          end
+          log_error("Could not seek to offset: #{e.message} after #{MAX_SEEK_RETRIES} retries", listener_metadata)
         end
 
         instrument('listener.start_handler', listener_metadata) do
@@ -50,7 +57,6 @@ module Deimos
 
       # :nodoc:
       def consume(payload, metadata)
-        puts "Got #{payload}"
         self.class.total_messages << {
           key: metadata[:key],
           payload: payload

data/lib/deimos/utils/schema_controller_mixin.rb

@@ -28,14 +28,18 @@ module Deimos
 
         # Indicate which schemas should be assigned to actions.
         # @param actions [Symbol]
+        # @param kwactions [String]
         # @param request [String]
         # @param response [String]
-        def schemas(*actions, request: nil, response: nil)
+        def schemas(*actions, request: nil, response: nil, **kwactions)
           actions.each do |action|
             request ||= action.to_s.titleize
             response ||= action.to_s.titleize
             schema_mapping[action.to_s] = { request: request, response: response }
           end
+          kwactions.each do |key, val|
+            schema_mapping[key.to_s] = { request: val, response: val }
+          end
         end
 
         # @return [Hash<Symbol, String>]

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.8.1-beta9'
+  VERSION = '1.8.4'
 end

data/spec/active_record_consumer_spec.rb

@@ -137,5 +137,18 @@ module ActiveRecordConsumerTest
       expect(Widget.find_by_test_id('id1').some_int).to eq(3)
       expect(Widget.find_by_test_id('id2').some_int).to eq(4)
     end
+
+    it 'should not create record of process_message returns false' do
+      MyConsumer.any_instance.stub(:process_message?).and_return(false)
+      expect(Widget.count).to eq(0)
+      test_consume_message(MyConsumer, {
+                             test_id: 'abc',
+                             some_int: 3,
+                             updated_at: 1.day.ago.to_i,
+                             some_datetime_int: Time.zone.now.to_i,
+                             timestamp: 2.minutes.ago.to_s
+                           }, { call_original: true, key: 5 })
+      expect(Widget.count).to eq(0)
+    end
   end
 end

data/spec/backends/db_spec.rb

@@ -43,6 +43,12 @@ each_db_config(Deimos::Backends::Db) do
     described_class.publish(producer_class: MyNoKeyProducer,
                             messages: [messages.first])
     expect(Deimos::KafkaMessage.count).to eq(4)
+  end
 
+  it 'should add messages with Hash keys with JSON encoding' do
+    described_class.publish(producer_class: MyProducer,
+                            messages: [build_message({ foo: 0 }, 'my-topic', { 'test_id' => 0 })])
+    expect(Deimos::KafkaMessage.count).to eq(1)
+    expect(Deimos::KafkaMessage.last.partition_key).to eq(%(---\ntest_id: 0\n))
   end
 end

data/spec/config/configuration_spec.rb

@@ -6,6 +6,14 @@ class MyConfigConsumer < Deimos::Consumer
   def consume
   end
 end
+
+# Mock consumer 2
+class MyConfigConsumer2 < Deimos::Consumer
+  # :no-doc:
+  def consume
+  end
+end
+
 describe Deimos, 'configuration' do
   it 'should configure with deprecated fields' do
     logger = Logger.new(nil)
@@ -171,6 +179,13 @@ describe Deimos, 'configuration' do
         offset_retention_time 13
         heartbeat_interval 13
       end
+      consumer do
+        disabled true
+        class_name 'MyConfigConsumer2'
+        schema 'blah2'
+        topic 'blah2'
+        group_id 'myconsumerid2'
+      end
     end
 
     expect(described_class.config.phobos_config).

data/spec/kafka_source_spec.rb

@@ -225,5 +225,88 @@ module KafkaSourceSpec
         expect(Deimos::KafkaMessage.count).to eq(0)
       end
     end
+
+    context 'with import hooks disabled' do
+      before(:each) do
+        # Dummy class we can include the mixin in. Has a backing table created
+        # earlier and has the import hook disabled
+        class WidgetNoImportHook < ActiveRecord::Base
+          include Deimos::KafkaSource
+          self.table_name = 'widgets'
+
+          # :nodoc:
+          def self.kafka_config
+            {
+              update: true,
+              delete: true,
+              import: false,
+              create: true
+            }
+          end
+
+          # :nodoc:
+          def self.kafka_producers
+            [WidgetProducer]
+          end
+        end
+        WidgetNoImportHook.reset_column_information
+      end
+
+      it 'should not fail when bulk-importing with existing records' do
+        widget1 = WidgetNoImportHook.create(widget_id: 1, name: 'Widget 1')
+        widget2 = WidgetNoImportHook.create(widget_id: 2, name: 'Widget 2')
+        widget1.name = 'New Widget No Import Hook 1'
+        widget2.name = 'New Widget No Import Hook 2'
+
+        expect {
+          WidgetNoImportHook.import([widget1, widget2], on_duplicate_key_update: %i(widget_id name))
+        }.not_to raise_error
+
+        expect('my-topic').not_to have_sent({
+                                              widget_id: 1,
+                                              name: 'New Widget No Import Hook 1',
+                                              id: widget1.id,
+                                              created_at: anything,
+                                              updated_at: anything
+                                            }, widget1.id)
+        expect('my-topic').not_to have_sent({
+                                              widget_id: 2,
+                                              name: 'New Widget No Import Hook 2',
+                                              id: widget2.id,
+                                              created_at: anything,
+                                              updated_at: anything
+                                            }, widget2.id)
+      end
+
+      it 'should not fail when mixing existing and new records' do
+        widget1 = WidgetNoImportHook.create(widget_id: 1, name: 'Widget 1')
+        expect('my-topic').to have_sent({
+                                          widget_id: 1,
+                                          name: 'Widget 1',
+                                          id: widget1.id,
+                                          created_at: anything,
+                                          updated_at: anything
+                                        }, widget1.id)
+
+        widget2 = WidgetNoImportHook.new(widget_id: 2, name: 'Widget 2')
+        widget1.name = 'New Widget 1'
+        WidgetNoImportHook.import([widget1, widget2], on_duplicate_key_update: %i(widget_id))
+        widgets = WidgetNoImportHook.all
+        expect('my-topic').not_to have_sent({
+                                              widget_id: 1,
+                                              name: 'New Widget 1',
+                                              id: widgets[0].id,
+                                              created_at: anything,
+                                              updated_at: anything
+                                            }, widgets[0].id)
+        expect('my-topic').not_to have_sent({
+                                              widget_id: 2,
+                                              name: 'Widget 2',
+                                              id: widgets[1].id,
+                                              created_at: anything,
+                                              updated_at: anything
+                                            }, widgets[1].id)
+      end
+    end
   end
 end

data/spec/producer_spec.rb

@@ -64,6 +64,14 @@ module ProducerTest
       end
       stub_const('MyErrorProducer', producer_class)
 
+      producer_class = Class.new(Deimos::Producer) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        topic nil
+        key_config none: true
+      end
+      stub_const('MyNoTopicProducer', producer_class)
+
     end
 
     it 'should fail on invalid message with error handler' do
@@ -102,6 +110,33 @@ module ProducerTest
       expect('my-topic').not_to have_sent('test_id' => 'foo2', 'some_int' => 123)
     end
 
+    it 'should allow setting the topic from publish_list' do
+      expect(described_class).to receive(:produce_batch).once.with(
+        Deimos::Backends::Test,
+        [
+          Deimos::Message.new({ 'test_id' => 'foo', 'some_int' => 123 },
+                              MyProducer,
+                              topic: 'a-new-topic',
+                              partition_key: 'foo',
+                              key: 'foo'),
+          Deimos::Message.new({ 'test_id' => 'bar', 'some_int' => 124 },
+                              MyProducer,
+                              topic: 'a-new-topic',
+                              partition_key: 'bar',
+                              key: 'bar')
+        ]
+      ).and_call_original
+
+      MyProducer.publish_list(
+        [{ 'test_id' => 'foo', 'some_int' => 123 },
+         { 'test_id' => 'bar', 'some_int' => 124 }],
+        topic: 'a-new-topic'
+      )
+      expect('a-new-topic').to have_sent('test_id' => 'foo', 'some_int' => 123)
+      expect('my-topic').not_to have_sent('test_id' => 'foo', 'some_int' => 123)
+      expect('my-topic').not_to have_sent('test_id' => 'foo2', 'some_int' => 123)
+    end
+
     it 'should add a message ID' do
       payload = { 'test_id' => 'foo',
                   'some_int' => 123,
@@ -201,6 +236,7 @@ module ProducerTest
     end
 
     it 'should encode the key' do
+      Deimos.configure { |c| c.producers.topic_prefix = nil }
       expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
       expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'my-topic-key')
       expect(MyProducer.encoder).to receive(:encode).with({
@@ -218,6 +254,21 @@ module ProducerTest
       )
     end
 
+    it 'should encode the key with topic prefix' do
+      Deimos.configure { |c| c.producers.topic_prefix = 'prefix.' }
+      expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'prefix.my-topic-key')
+      expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'prefix.my-topic-key')
+      expect(MyProducer.encoder).to receive(:encode).with({  'test_id' => 'foo',
+                                                             'some_int' => 123 },
+                                                          { topic: 'prefix.my-topic-value' })
+      expect(MyProducer.encoder).to receive(:encode).with({  'test_id' => 'bar',
+                                                             'some_int' => 124 },
+                                                          { topic: 'prefix.my-topic-value' })
+
+      MyProducer.publish_list([{ 'test_id' => 'foo', 'some_int' => 123 },
+                               { 'test_id' => 'bar', 'some_int' => 124 }])
+    end
+
     it 'should not encode with plaintext key' do
       expect(MyNonEncodedProducer.key_encoder).not_to receive(:encode_key)
 
@@ -269,6 +320,31 @@ module ProducerTest
       )
     end
 
+    it 'should raise error if blank topic is passed in explicitly' do
+      expect {
+        MyProducer.publish_list(
+          [{  'test_id' => 'foo',
+              'some_int' => 123 },
+           {  'test_id' => 'bar',
+              'some_int' => 124 }],
+          topic: ''
+        )
+      }.to raise_error(RuntimeError,
+                       'Topic not specified. Please specify the topic.')
+    end
+
+    it 'should raise error if the producer has not been initialized with a topic' do
+      expect {
+        MyNoTopicProducer.publish_list(
+          [{  'test_id' => 'foo',
+              'some_int' => 123 },
+           {  'test_id' => 'bar',
+              'some_int' => 124 }]
+        )
+      }.to raise_error(RuntimeError,
+                       'Topic not specified. Please specify the topic.')
+    end
+
     it 'should error with nothing set' do
       expect {
         MyErrorProducer.publish_list(

data/spec/schema_backends/avro_base_shared.rb

@@ -42,6 +42,20 @@ RSpec.shared_examples_for('an Avro backend') do
         {
           'name' => 'union-int-field',
           'type' => %w(null int)
+        },
+        {
+          'name' => 'timestamp-millis-field',
+          'type' => {
+            'type' => 'long',
+            'logicalType' => 'timestamp-millis'
+          }
+        },
+        {
+          'name' => 'timestamp-micros-field',
+          'type' => {
+            'type' => 'long',
+            'logicalType' => 'timestamp-micros'
+          }
         }
       ]
     }
@@ -95,7 +109,9 @@ RSpec.shared_examples_for('an Avro backend') do
         'string-field' => 'hi mom',
         'boolean-field' => true,
         'union-field' => nil,
-        'union-int-field' => nil
+        'union-int-field' => nil,
+        'timestamp-millis-field' => Time.utc(2020, 11, 12, 13, 14, 15, 909_090),
+        'timestamp-micros-field' => Time.utc(2020, 11, 12, 13, 14, 15, 909_090)
       }
     end
 
@@ -169,6 +185,15 @@ RSpec.shared_examples_for('an Avro backend') do
       expect(result['union-field']).to eq('itsme')
     end
 
+    it 'should not convert timestamp-millis' do
+      result = backend.coerce(payload)
+      expect(result['timestamp-millis-field']).to eq(Time.utc(2020, 11, 12, 13, 14, 15, 909_090))
+    end
+
+    it 'should not convert timestamp-micros' do
+      result = backend.coerce(payload)
+      expect(result['timestamp-micros-field']).to eq(Time.utc(2020, 11, 12, 13, 14, 15, 909_090))
+    end
   end
 
 end

data/spec/schemas/com/my-namespace/request/CreateTopic.avsc

@@ -0,0 +1,11 @@
+{
+  "namespace": "com.my-namespace.request",
+  "name": "CreateTopic",
+  "type": "record",
+  "fields": [
+    {
+      "name": "request_id",
+      "type": "string"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/response/CreateTopic.avsc

@@ -0,0 +1,11 @@
+{
+  "namespace": "com.my-namespace.response",
+  "name": "CreateTopic",
+  "type": "record",
+  "fields": [
+    {
+      "name": "response_id",
+      "type": "string"
+    }
+  ]
+}

data/spec/utils/db_producer_spec.rb

@@ -96,7 +96,32 @@ each_db_config(Deimos::Utils::DbProducer) do
       expect(phobos_producer).to have_received(:publish_list).with(['A'] * 100).once
       expect(phobos_producer).to have_received(:publish_list).with(['A'] * 10).once
       expect(phobos_producer).to have_received(:publish_list).with(['A']).once
+    end
+
+    it 'should not resend batches of sent messages' do
+      allow(phobos_producer).to receive(:publish_list) do |group|
+        raise Kafka::BufferOverflow if group.any?('A') && group.size >= 1000
+        raise Kafka::BufferOverflow if group.any?('BIG') && group.size >= 10
+      end
+      allow(Deimos.config.metrics).to receive(:increment)
+      batch = ['A'] * 450 + ['BIG'] * 550
+      producer.produce_messages(batch)
+
+      expect(phobos_producer).to have_received(:publish_list).with(batch)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 100).exactly(4).times
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 50 + ['BIG'] * 50)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 10).exactly(5).times
+      expect(phobos_producer).to have_received(:publish_list).with(['BIG'] * 1).exactly(550).times
 
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 100).exactly(4).times
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 10).exactly(5).times
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 1).exactly(550).times
     end
 
     describe '#compact_messages' do
@@ -289,6 +314,8 @@ each_db_config(Deimos::Utils::DbProducer) do
           message: "mess#{i}",
           partition_key: "key#{i}"
         )
+      end
+      (5..8).each do |i|
         Deimos::KafkaMessage.create!(
           id: i,
           topic: 'my-topic2',

data/spec/utils/inline_consumer_spec.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+describe Deimos::Utils::SeekListener do
+
+  describe '#start_listener' do
+    let(:consumer) { instance_double(Kafka::Consumer) }
+    let(:handler) { class_double(Deimos::Utils::MessageBankHandler) }
+
+    before(:each) do
+      allow(handler).to receive(:start)
+      allow(consumer).to receive(:subscribe)
+      allow_any_instance_of(Phobos::Listener).to receive(:create_kafka_consumer).and_return(consumer)
+      allow_any_instance_of(Kafka::Client).to receive(:last_offset_for).and_return(100)
+      stub_const('Deimos::Utils::SeekListener::MAX_SEEK_RETRIES', 2)
+    end
+
+    it 'should seek offset' do
+      allow(consumer).to receive(:seek)
+      expect(consumer).to receive(:seek).once
+      seek_listener = described_class.new({ handler: handler, group_id: 999, topic: 'test_topic' })
+      seek_listener.start_listener
+    end
+
+    it 'should retry on errors when seeking offset' do
+      allow(consumer).to receive(:seek).and_raise(StandardError)
+      expect(consumer).to receive(:seek).twice
+      seek_listener = described_class.new({ handler: handler, group_id: 999, topic: 'test_topic' })
+      seek_listener.start_listener
+    end
+  end
+end

data/spec/utils/schema_controller_mixin_spec.rb

@@ -17,6 +17,7 @@ RSpec.describe Deimos::Utils::SchemaControllerMixin, type: :controller do
     request_namespace 'com.my-namespace.request'
     response_namespace 'com.my-namespace.response'
     schemas :index, :show
+    schemas create: 'CreateTopic'
     schemas :update, request: 'UpdateRequest', response: 'UpdateResponse'
 
     # :nodoc:
@@ -29,6 +30,11 @@ RSpec.describe Deimos::Utils::SchemaControllerMixin, type: :controller do
       render_schema({ 'response_id' => payload[:request_id] + ' dad' })
     end
 
+    # :nodoc:
+    def create
+      render_schema({ 'response_id' => payload[:request_id] + ' bro' })
+    end
+
     # :nodoc:
     def update
       render_schema({ 'update_response_id' => payload[:update_request_id] + ' sis' })
@@ -65,4 +71,14 @@ RSpec.describe Deimos::Utils::SchemaControllerMixin, type: :controller do
     expect(response_backend.decode(response.body)).to eq({ 'update_response_id' => 'hi sis' })
   end
 
+  it 'should render the correct response for create' do
+    request_backend = Deimos.schema_backend(schema: 'CreateTopic',
+                                            namespace: 'com.my-namespace.request')
+    response_backend = Deimos.schema_backend(schema: 'CreateTopic',
+                                             namespace: 'com.my-namespace.response')
+    request.content_type = 'avro/binary'
+    post :create, params: { id: 1 }, body: request_backend.encode({ 'request_id' => 'hi' })
+    expect(response_backend.decode(response.body)).to eq({ 'response_id' => 'hi bro' })
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.8.1.pre.beta9
+  version: 1.8.4
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-27 00:00:00.000000000 Z
+date: 2020-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -348,7 +348,9 @@ files:
 - docs/ARCHITECTURE.md
 - docs/CONFIGURATION.md
 - docs/DATABASE_BACKEND.md
+- docs/INTEGRATION_TESTS.md
 - docs/PULL_REQUEST_TEMPLATE.md
+- docs/UPGRADING.md
 - lib/deimos.rb
 - lib/deimos/active_record_consume/batch_consumption.rb
 - lib/deimos/active_record_consume/batch_slicer.rb
@@ -453,14 +455,17 @@ files:
 - spec/schemas/com/my-namespace/Wibble.avsc
 - spec/schemas/com/my-namespace/Widget.avsc
 - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
+- spec/schemas/com/my-namespace/request/CreateTopic.avsc
 - spec/schemas/com/my-namespace/request/Index.avsc
 - spec/schemas/com/my-namespace/request/UpdateRequest.avsc
+- spec/schemas/com/my-namespace/response/CreateTopic.avsc
 - spec/schemas/com/my-namespace/response/Index.avsc
 - spec/schemas/com/my-namespace/response/UpdateResponse.avsc
 - spec/spec_helper.rb
 - spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
 - spec/utils/deadlock_retry_spec.rb
+- spec/utils/inline_consumer_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
 - spec/utils/schema_controller_mixin_spec.rb
@@ -483,9 +488,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.1.3
 signing_key:
@@ -534,14 +539,17 @@ test_files:
 - spec/schemas/com/my-namespace/Wibble.avsc
 - spec/schemas/com/my-namespace/Widget.avsc
 - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
+- spec/schemas/com/my-namespace/request/CreateTopic.avsc
 - spec/schemas/com/my-namespace/request/Index.avsc
 - spec/schemas/com/my-namespace/request/UpdateRequest.avsc
+- spec/schemas/com/my-namespace/response/CreateTopic.avsc
 - spec/schemas/com/my-namespace/response/Index.avsc
 - spec/schemas/com/my-namespace/response/UpdateResponse.avsc
 - spec/spec_helper.rb
 - spec/utils/db_poller_spec.rb
 - spec/utils/db_producer_spec.rb
 - spec/utils/deadlock_retry_spec.rb
+- spec/utils/inline_consumer_spec.rb
 - spec/utils/lag_reporter_spec.rb
 - spec/utils/platform_schema_validation_spec.rb
 - spec/utils/schema_controller_mixin_spec.rb