deimos-ruby 1.8.1.pre.beta6 → 1.8.2.pre.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d38b0803c074ff89dd8ff166ed8de2dcacb2ecaae46cb321abc0de630498e849
-  data.tar.gz: efecfba1d4d76a83270534ab7687c344253b2e4a7765dde9d42b6959083171db
+  metadata.gz: b773de54759babb229e8f3bc6b42931dab7695b7197d601ad4ae08015c0b401f
+  data.tar.gz: 4af1aa98ffdfc013c4cac02fa316371ad138ec0d807ed58426a7ad57d6c1e719
 SHA512:
-  metadata.gz: d435a7c9f130650f7f9849554f2d0a71b87a9f8bddca2f542b94848ca13e8fe3084aa095f19eadd581a7af42f43511509ae9dc06bbb69982da34463304945a69
-  data.tar.gz: 9cfd22453a8a59d6c74a46bf41c6feb874bf125fb335c4b084df04f681bb9f3d2b03a8295a18c86166c194e592855d8b811bad39c29565002ec6dd6100edd589
+  metadata.gz: 319d01a4c42c5efe72bd4959fb57548102c3f2b4305f3d94510747fd372d516ebad3e34ff74f9e6744641728efce9eb7a20f2d89c37d6bd90847253698a05238
+  data.tar.gz: 9e6999cca87ca448167a9e61a0bfa5914b36fd74c053f132bd02ab5200be840984546469affccfd07ce77d4d098a2096b13e42febd4ce7059ec882c436580cd8

data/CHANGELOG.md

@@ -7,7 +7,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.8.2-beta2 - 2020-09-15
+
+### Features :star:
+
+- Add details on using schema backend directly in README.
+- Default to the provided schema if topic is not provided when
+  encoding to `AvroSchemaRegistry`.
+- Add mapping syntax for the `schema` call in `SchemaControllerMixin`.
+
+## 1.8.2-beta1 - 2020-09-09
+
+### Features :star:
+
+- Added the ability to specify the topic for `publish`
+and `publish_list` in a producer
+
+## 1.8.1-beta9 - 2020-08-27
+
+### Fixes :wrench:
+- Moved the TestHelpers hook to `before(:suite)` to allow for
+  overriding e.g. in integration tests.
+
+## 1.8.1-beta7 - 2020-08-25
+
+### Fixes :wrench:
+- Fix for crash when sending pending metrics with DB producer.
+- Fix for compacting messages if all the keys are already unique
+  (fixes [#75](https://github.com/flipp-oss/deimos/issues/75))
+
 ## 1.8.1-beta6 - 2020-08-13
+
+### Fixes :wrench:
+
 - Fix for consuming nil payloads with Ruby 2.3.
 
 ## 1.8.1-beta5 - 2020-08-13

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.8.1.pre.beta6)
+    deimos-ruby (1.8.2.pre.beta2)
       avro_turf (~> 0.11)
       phobos (~> 1.9)
       ruby-kafka (~> 0.7)

data/README.md

@@ -11,6 +11,7 @@ a useful toolbox of goodies for Ruby-based Kafka development.
 Built on Phobos and hence Ruby-Kafka.
 
 <!--ts-->
+   * [Additional Documentation](#additional-documentation)
    * [Installation](#installation)
    * [Versioning](#versioning)
    * [Configuration](#configuration)
@@ -29,9 +30,19 @@ Built on Phobos and hence Ruby-Kafka.
    * [Metrics](#metrics)
    * [Testing](#testing)
         * [Integration Test Helpers](#integration-test-helpers)
+   * [Utilities](#utilities)
    * [Contributing](#contributing) 
 <!--te-->
 
+# Additional Documentation
+
+Please see the following for further information not covered by this readme:
+
+* [Architecture Design](docs/ARCHITECTURE.md)
+* [Configuration Reference](docs/CONFIGURATION.md)
+* [Database Backend Feature](docs/DATABASE_BACKEND.md)
+* [Upgrading Deimos](docs/UPGRADING.md)
+
 # Installation
 
 Add this line to your application's Gemfile:
@@ -108,6 +119,7 @@ class MyProducer < Deimos::Producer
         'some-key2' => an_object.bar
       }
       # You can also publish an array with self.publish_list(payloads)
+      # You may specify the topic here with self.publish(payload, topic: 'my-topic')
       self.publish(payload)
     end
     
@@ -481,6 +493,12 @@ class WhateverController < ApplicationController
   # will look for: my.namespace.requests.Index.avsc
   #                my.namespace.responses.Index.avsc 
   
+  # Can use mapping to change the schema but keep the namespaces,
+  # i.e. use the same schema name across the two namespaces
+  schemas create: 'CreateTopic'
+  # will look for: my.namespace.requests.CreateTopic.avsc
+  #                my.namespace.responses.CreateTopic.avsc 
+
   # If all routes use the default, you can add them all at once
   schemas :index, :show, :update
 
@@ -921,6 +939,17 @@ Deimos::TestHelpers.schemas_compatible?(schema1, schema2)
 
 ### Integration Test Helpers
 
+When running integration tests, you'll want to override the default test helper settings:
+
+```ruby
+config.before(:each, :my_integration_metadata) do
+  Deimos.configure do
+    producers.backend :kafka
+    schema.backend :avro_schema_registry
+  end
+end
+```
+
 You can use the `InlineConsumer` class to help with integration testing,
 with a full external Kafka running.
 
@@ -977,13 +1006,24 @@ Deimos::Utils::InlineConsumer.get_messages_for(
 )
 ```
 
+## Utilities
+
+You can use your configured schema backend directly if you want to
+encode and decode payloads outside of the context of sending messages.
+
+```ruby
+backend = Deimos.schema_backend(schema: 'MySchema', namespace: 'com.my-namespace')
+encoded = backend.encode(my_payload)
+decoded = backend.decode(my_encoded_payload)
+coerced = backend.coerce(my_payload) # coerce to correct types
+backend.validate(my_payload) # throws an error if not valid
+fields = backend.schema_fields # list of fields defined in the schema
+```
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/flipp-oss/deimos .
 
-We have more information on the [internal architecture](docs/ARCHITECTURE.md) of Deimos
-for contributors!
-
 ### Linting
 
 Deimos uses Rubocop to lint the code. Please run Rubocop on your code 

data/docs/UPGRADING.md

@@ -0,0 +1,128 @@
+# Upgrading Deimos
+
+## Upgrading from < 1.5.0 to >= 1.5.0
+
+If you are using Confluent's schema registry to Avro-encode your
+messages, you will need to manually include the `avro_turf` gem
+in your Gemfile now.
+
+This update changes how to interact with Deimos's schema classes.
+Although these are meant to be internal, they are still "public"
+and can be used by calling code.
+
+Before 1.5.0:
+
+```ruby
+encoder = Deimos::AvroDataEncoder.new(schema: 'MySchema',
+                                      namespace: 'com.my-namespace')
+encoder.encode(my_payload)
+
+decoder = Deimos::AvroDataDecoder.new(schema: 'MySchema',
+                                      namespace: 'com.my-namespace')
+decoder.decode(my_payload)
+```
+
+After 1.5.0:
+```ruby
+backend = Deimos.schema_backend(schema: 'MySchema', namespace: 'com.my-namespace')
+backend.encode(my_payload)
+backend.decode(my_payload)
+```
+
+The two classes are different and if you are using them to e.g.
+inspect Avro schema fields, please look at the source code for the following:
+* `Deimos::SchemaBackends::Base`
+* `Deimos::SchemaBackends::AvroBase`
+* `Deimos::SchemaBackends::AvroSchemaRegistry`
+
+Deprecated `Deimos::TestHelpers.sent_messages` in favor of
+`Deimos::Backends::Test.sent_messages`.
+
+## Upgrading from < 1.4.0 to >= 1.4.0 
+
+Previously, configuration was handled as follows:
+* Kafka configuration, including listeners, lived in `phobos.yml`
+* Additional Deimos configuration would live in an initializer, e.g. `kafka.rb`
+* Producer and consumer configuration lived in each individual producer and consumer
+
+As of 1.4.0, all configuration is centralized in one initializer
+file, using default configuration.
+
+Before 1.4.0:
+```yaml
+# config/phobos.yml
+logger:
+  file: log/phobos.log
+  level: debug
+  ruby_kafka:
+    level: debug
+
+kafka:
+  client_id: phobos
+  connect_timeout: 15
+  socket_timeout: 15
+
+producer:
+  ack_timeout: 5
+  required_acks: :all
+  ...
+
+listeners:
+  - handler: ConsumerTest::MyConsumer
+    topic: my_consume_topic
+    group_id: my_group_id
+  - handler: ConsumerTest::MyBatchConsumer
+    topic: my_batch_consume_topic
+    group_id: my_batch_group_id
+    delivery: inline_batch
+```
+
+```ruby
+# kafka.rb
+Deimos.configure do |config|
+  config.reraise_consumer_errors = true
+  config.logger = Rails.logger
+  ...
+end
+
+# my_consumer.rb
+class ConsumerTest::MyConsumer < Deimos::Producer
+  namespace 'com.my-namespace'
+  schema 'MySchema'
+  topic 'MyTopic'
+  key_config field: :id
+end
+```
+
+After 1.4.0:
+```ruby
+kafka.rb
+Deimos.configure do
+  logger Rails.logger
+  kafka do
+    client_id 'phobos'
+    connect_timeout 15
+    socket_timeout 15
+  end
+  producers.ack_timeout 5
+  producers.required_acks :all
+  ...
+  consumer do
+    class_name 'ConsumerTest::MyConsumer'
+    topic 'my_consume_topic'
+    group_id 'my_group_id' 
+    namespace 'com.my-namespace'
+    schema 'MySchema'
+    topic 'MyTopic'
+    key_config field: :id
+  end
+  ...
+end
+```
+
+Note that the old configuration way *will* work if you set
+`config.phobos_config_file = "config/phobos.yml"`. You will
+get a number of deprecation notices, however. You can also still
+set the topic, namespace, etc. on the producer/consumer class,
+but it's much more convenient to centralize these configs
+in one place to see what your app does.

data/lib/deimos/producer.rb

@@ -87,8 +87,9 @@ module Deimos
 
       # Publish the payload to the topic.
       # @param payload [Hash] with an optional payload_key hash key.
-      def publish(payload)
-        publish_list([payload])
+      # @param topic [String] if specifying the topic
+      def publish(payload, topic: self.topic)
+        publish_list([payload], topic: topic)
       end
 
       # Publish a list of messages.
@@ -97,7 +98,8 @@ module Deimos
       # whether to publish synchronously.
       # @param force_send [Boolean] if true, ignore the configured backend
       # and send immediately to Kafka.
-      def publish_list(payloads, sync: nil, force_send: false)
+      # @param topic [String] if specifying the topic
+      def publish_list(payloads, sync: nil, force_send: false, topic: self.topic)
         return if Deimos.config.kafka.seed_brokers.blank? ||
                   Deimos.config.producers.disabled ||
                   Deimos.producers_disabled?(self)
@@ -110,7 +112,7 @@ module Deimos
           payloads: payloads
         ) do
           messages = Array(payloads).map { |p| Deimos::Message.new(p, self) }
-          messages.each(&method(:_process_message))
+          messages.each { |m| _process_message(m, topic) }
           messages.in_groups_of(MAX_BATCH_SIZE, false) do |batch|
             self.produce_batch(backend_class, batch)
           end
@@ -163,7 +165,8 @@ module Deimos
     private
 
       # @param message [Message]
-      def _process_message(message)
+      # @param topic [String]
+      def _process_message(message, topic)
         # this violates the Law of Demeter but it has to happen in a very
         # specific order and requires a bunch of methods on the producer
         # to work correctly.
@@ -175,7 +178,7 @@ module Deimos
         message.payload = nil if message.payload.blank?
         message.coerce_fields(encoder)
         message.encoded_key = _encode_key(message.key)
-        message.topic = self.topic
+        message.topic = topic
         message.encoded_payload = if message.payload.nil?
                                     nil
                                   else

data/lib/deimos/schema_backends/avro_schema_registry.rb

@@ -15,7 +15,7 @@ module Deimos
 
       # @override
       def encode_payload(payload, schema: nil, topic: nil)
-        avro_turf_messaging.encode(payload, schema_name: schema, subject: topic)
+        avro_turf_messaging.encode(payload, schema_name: schema, subject: topic || schema)
       end
 
     private

data/lib/deimos/test_helpers.rb

@@ -30,18 +30,19 @@ module Deimos
             d_config.consumers.reraise_errors = true
             d_config.kafka.seed_brokers ||= ['test_broker']
             d_config.schema.backend = Deimos.schema_backend_class.mock_backend
+            d_config.producers.backend = :test
           end
         end
-      end
 
-      before(:each) do
-        client = double('client').as_null_object
-        allow(client).to receive(:time) do |*_args, &block|
-          block.call
+        config.before(:each) do
+          client = double('client').as_null_object
+          allow(client).to receive(:time) do |*_args, &block|
+            block.call
+          end
+          Deimos::Backends::Test.sent_messages.clear
         end
-        Deimos.configure { |c| c.producers.backend = :test }
-        Deimos::Backends::Test.sent_messages.clear
       end
+
     end
 
     # @deprecated

data/lib/deimos/utils/db_producer.rb

@@ -161,7 +161,7 @@ module Deimos
           # the oldest message, or the last time we processed, whichever comes
           # last.
           if message_record
-            record_earliest = record.earliest
+            record_earliest = message_record.earliest
             # SQLite gives a string here
             if record_earliest.is_a?(String)
               record_earliest = Time.zone.parse(record_earliest)
@@ -231,7 +231,7 @@ module Deimos
         return batch if config.compact_topics != :all &&
                         !config.compact_topics.include?(topic)
 
-        batch.reverse.uniq!(&:key).reverse!
+        batch.reverse.uniq(&:key).reverse!
       end
     end
   end

data/lib/deimos/utils/schema_controller_mixin.rb

@@ -28,14 +28,18 @@ module Deimos
 
         # Indicate which schemas should be assigned to actions.
         # @param actions [Symbol]
+        # @param kwactions [String]
         # @param request [String]
         # @param response [String]
-        def schemas(*actions, request: nil, response: nil)
+        def schemas(*actions, request: nil, response: nil, **kwactions)
           actions.each do |action|
             request ||= action.to_s.titleize
             response ||= action.to_s.titleize
             schema_mapping[action.to_s] = { request: request, response: response }
           end
+          kwactions.each do |key, val|
+            schema_mapping[key.to_s] = { request: val, response: val }
+          end
         end
 
         # @return [Hash<Symbol, String>]

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.8.1-beta6'
+  VERSION = '1.8.2-beta2'
 end

data/spec/consumer_spec.rb

@@ -34,8 +34,8 @@ module ConsumerTest
 
     it 'should consume a nil message' do
       test_consume_message(MyConsumer, nil) do |payload, _metadata|
-                             expect(payload).to be_nil
-                           end
+        expect(payload).to be_nil
+      end
     end
 
     it 'should consume a message idempotently' do

data/spec/producer_spec.rb

@@ -102,6 +102,33 @@ module ProducerTest
       expect('my-topic').not_to have_sent('test_id' => 'foo2', 'some_int' => 123)
     end
 
+    it 'should allow setting the topic from publish_list' do
+      expect(described_class).to receive(:produce_batch).once.with(
+        Deimos::Backends::Test,
+        [
+          Deimos::Message.new({ 'test_id' => 'foo', 'some_int' => 123 },
+                              MyProducer,
+                              topic: 'a-new-topic',
+                              partition_key: 'foo',
+                              key: 'foo'),
+          Deimos::Message.new({ 'test_id' => 'bar', 'some_int' => 124 },
+                              MyProducer,
+                              topic: 'a-new-topic',
+                              partition_key: 'bar',
+                              key: 'bar')
+        ]
+      ).and_call_original
+
+      MyProducer.publish_list(
+        [{ 'test_id' => 'foo', 'some_int' => 123 },
+         { 'test_id' => 'bar', 'some_int' => 124 }],
+        topic: 'a-new-topic'
+      )
+      expect('a-new-topic').to have_sent('test_id' => 'foo', 'some_int' => 123)
+      expect('my-topic').not_to have_sent('test_id' => 'foo', 'some_int' => 123)
+      expect('my-topic').not_to have_sent('test_id' => 'foo2', 'some_int' => 123)
+    end
+
     it 'should add a message ID' do
       payload = { 'test_id' => 'foo',
                   'some_int' => 123,

data/spec/schemas/com/my-namespace/request/CreateTopic.avsc

@@ -0,0 +1,11 @@
+{
+  "namespace": "com.my-namespace.request",
+  "name": "CreateTopic",
+  "type": "record",
+  "fields": [
+    {
+      "name": "request_id",
+      "type": "string"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/response/CreateTopic.avsc

@@ -0,0 +1,11 @@
+{
+  "namespace": "com.my-namespace.response",
+  "name": "CreateTopic",
+  "type": "record",
+  "fields": [
+    {
+      "name": "response_id",
+      "type": "string"
+    }
+  ]
+}

data/spec/spec_helper.rb

@@ -183,6 +183,7 @@ RSpec.configure do |config|
   config.before(:each) do
     Deimos.config.reset!
     Deimos.configure do |deimos_config|
+      deimos_config.producers.backend = :test
       deimos_config.phobos_config_file = File.join(File.dirname(__FILE__), 'phobos.yml')
       deimos_config.schema.path = File.join(File.expand_path(__dir__), 'schemas')
       deimos_config.consumers.reraise_errors = true

data/spec/utils/db_producer_spec.rb

@@ -155,6 +155,10 @@ each_db_config(Deimos::Utils::DbProducer) do
         Deimos.configure { |c| c.db_producer.compact_topics = [] }
       end
 
+      it 'should compact messages when all messages are unique' do
+        Deimos.configure { |c| c.db_producer.compact_topics = %w(my-topic my-topic2) }
+        expect(producer.compact_messages(deduped_batch)).to eq(deduped_batch)
+      end
     end
   end
 

data/spec/utils/schema_controller_mixin_spec.rb

@@ -17,6 +17,7 @@ RSpec.describe Deimos::Utils::SchemaControllerMixin, type: :controller do
     request_namespace 'com.my-namespace.request'
     response_namespace 'com.my-namespace.response'
     schemas :index, :show
+    schemas create: 'CreateTopic'
     schemas :update, request: 'UpdateRequest', response: 'UpdateResponse'
 
     # :nodoc:
@@ -29,6 +30,11 @@ RSpec.describe Deimos::Utils::SchemaControllerMixin, type: :controller do
       render_schema({ 'response_id' => payload[:request_id] + ' dad' })
     end
 
+    # :nodoc:
+    def create
+      render_schema({ 'response_id' => payload[:request_id] + ' bro' })
+    end
+
     # :nodoc:
     def update
       render_schema({ 'update_response_id' => payload[:update_request_id] + ' sis' })
@@ -65,4 +71,14 @@ RSpec.describe Deimos::Utils::SchemaControllerMixin, type: :controller do
     expect(response_backend.decode(response.body)).to eq({ 'update_response_id' => 'hi sis' })
   end
 
+  it 'should render the correct response for create' do
+    request_backend = Deimos.schema_backend(schema: 'CreateTopic',
+                                            namespace: 'com.my-namespace.request')
+    response_backend = Deimos.schema_backend(schema: 'CreateTopic',
+                                             namespace: 'com.my-namespace.response')
+    request.content_type = 'avro/binary'
+    post :create, params: { id: 1 }, body: request_backend.encode({ 'request_id' => 'hi' })
+    expect(response_backend.decode(response.body)).to eq({ 'response_id' => 'hi bro' })
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.8.1.pre.beta6
+  version: 1.8.2.pre.beta2
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-13 00:00:00.000000000 Z
+date: 2020-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -349,6 +349,7 @@ files:
 - docs/CONFIGURATION.md
 - docs/DATABASE_BACKEND.md
 - docs/PULL_REQUEST_TEMPLATE.md
+- docs/UPGRADING.md
 - lib/deimos.rb
 - lib/deimos/active_record_consume/batch_consumption.rb
 - lib/deimos/active_record_consume/batch_slicer.rb
@@ -453,8 +454,10 @@ files:
 - spec/schemas/com/my-namespace/Wibble.avsc
 - spec/schemas/com/my-namespace/Widget.avsc
 - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
+- spec/schemas/com/my-namespace/request/CreateTopic.avsc
 - spec/schemas/com/my-namespace/request/Index.avsc
 - spec/schemas/com/my-namespace/request/UpdateRequest.avsc
+- spec/schemas/com/my-namespace/response/CreateTopic.avsc
 - spec/schemas/com/my-namespace/response/Index.avsc
 - spec/schemas/com/my-namespace/response/UpdateResponse.avsc
 - spec/spec_helper.rb
@@ -487,7 +490,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.1.3
+rubygems_version: 3.1.2
 signing_key:
 specification_version: 4
 summary: Kafka libraries for Ruby.
@@ -534,8 +537,10 @@ test_files:
 - spec/schemas/com/my-namespace/Wibble.avsc
 - spec/schemas/com/my-namespace/Widget.avsc
 - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
+- spec/schemas/com/my-namespace/request/CreateTopic.avsc
 - spec/schemas/com/my-namespace/request/Index.avsc
 - spec/schemas/com/my-namespace/request/UpdateRequest.avsc
+- spec/schemas/com/my-namespace/response/CreateTopic.avsc
 - spec/schemas/com/my-namespace/response/Index.avsc
 - spec/schemas/com/my-namespace/response/UpdateResponse.avsc
 - spec/spec_helper.rb