deimos-ruby 1.16.1 → 1.16.4

data/lib/deimos/utils/db_producer.rb

@@ -8,8 +8,11 @@ module Deimos
       include Phobos::Producer
       attr_accessor :id, :current_topic
 
+      # @return [Integer]
       BATCH_SIZE = 1000
+      # @return [Integer]
       DELETE_BATCH_SIZE = 10
+      # @return [Integer]
       MAX_DELETE_ATTEMPTS = 3
 
       # @param logger [Logger]
@@ -19,12 +22,13 @@ module Deimos
         @logger.push_tags("DbProducer #{@id}") if @logger.respond_to?(:push_tags)
       end
 
-      # @return [Deimos::DbProducerConfig]
+      # @return [FigTree]
       def config
         Deimos.config.db_producer
       end
 
       # Start the poll.
+      # @return [void]
       def start
         @logger.info('Starting...')
         @signal_to_stop = false
@@ -40,12 +44,14 @@ module Deimos
       end
 
       # Stop the poll.
+      # @return [void]
       def stop
         @logger.info('Received signal to stop')
         @signal_to_stop = true
       end
 
       # Complete one loop of processing all messages in the DB.
+      # @return [void]
       def process_next_messages
         topics = retrieve_topics
         @logger.info("Found topics: #{topics}")
@@ -80,6 +86,7 @@ module Deimos
       end
 
       # Process a single batch in a topic.
+      # @return [void]
       def process_topic_batch
         messages = retrieve_messages
         return false if messages.empty?
@@ -111,6 +118,7 @@ module Deimos
       end
 
       # @param messages [Array<Deimos::KafkaMessage>]
+      # @return [void]
       def delete_messages(messages)
         attempts = 1
         begin
@@ -137,6 +145,7 @@ module Deimos
       end
 
       # @param messages [Array<Deimos::KafkaMessage>]
+      # @return [void]
       def log_messages(messages)
         return if config.log_topics != :all && !config.log_topics.include?(@current_topic)
 
@@ -146,7 +155,8 @@ module Deimos
         end
       end
 
-      # Send metrics to Datadog.
+      # Send metrics related to pending messages.
+      # @return [void]
       def send_pending_metrics
         metrics = Deimos.config.metrics
         return unless metrics
@@ -185,6 +195,7 @@ module Deimos
       # Shut down the sync producer if we have to. Phobos will automatically
       # create a new one. We should call this if the producer can be in a bad
       # state and e.g. we need to clear the buffer.
+      # @return [void]
       def shutdown_producer
         if self.class.producer.respond_to?(:sync_producer_shutdown) # Phobos 1.8.3
           self.class.producer.sync_producer_shutdown
@@ -194,6 +205,7 @@ module Deimos
       # Produce messages in batches, reducing the size 1/10 if the batch is too
       # large. Does not retry batches of messages that have already been sent.
       # @param batch [Array<Hash>]
+      # @return [void]
       def produce_messages(batch)
         batch_size = batch.size
         current_index = 0

data/lib/deimos/utils/deadlock_retry.rb

@@ -7,9 +7,11 @@ module Deimos
     class DeadlockRetry
       class << self
         # Maximum number of times to retry the block after encountering a deadlock
+        # @return [Integer]
         RETRY_COUNT = 2
 
         # Need to match on error messages to support older Rails versions
+        # @return [Array<String>]
         DEADLOCK_MESSAGES = [
           # MySQL
           'Deadlock found when trying to get lock',
@@ -28,6 +30,7 @@ module Deimos
         # from retrying at the same time.
         # @param tags [Array] Tags to attach when logging and reporting metrics.
         # @yield Yields to the block that may deadlock.
+        # @return [void]
         def wrap(tags=[])
           count = RETRY_COUNT
 

data/lib/deimos/utils/inline_consumer.rb

@@ -6,10 +6,12 @@ module Deimos
   module Utils
     # Listener that can seek to get the last X messages in a topic.
     class SeekListener < Phobos::Listener
+      # @return [Integer]
       MAX_SEEK_RETRIES = 3
+      # @return [Integer]
       attr_accessor :num_messages
 
-      # :nodoc:
+      # @return [void]
       def start_listener
         @num_messages ||= 10
         @consumer = create_kafka_consumer
@@ -45,17 +47,20 @@ module Deimos
 
       cattr_accessor :total_messages
 
-      # @param klass [Class < Deimos::Consumer]
+      # @param klass [Class<Deimos::Consumer>]
+      # @return [void]
       def self.config_class=(klass)
         self.config.merge!(klass.config)
       end
 
-      # :nodoc:
+      # @param _kafka_client [Kafka::Client]
+      # @return [void]
       def self.start(_kafka_client)
         self.total_messages = []
       end
 
-      # :nodoc:
+      # @param payload [Hash]
+      # @param metadata [Hash]
       def consume(payload, metadata)
         self.class.total_messages << {
           key: metadata[:key],
@@ -66,18 +71,20 @@ module Deimos
 
     # Class which can process/consume messages inline.
     class InlineConsumer
+      # @return [Integer]
       MAX_MESSAGE_WAIT_TIME = 1.second
+      # @return [Integer]
       MAX_TOPIC_WAIT_TIME = 10.seconds
 
       # Get the last X messages from a topic. You can specify a subclass of
       # Deimos::Consumer or Deimos::Producer, or provide the
       # schema, namespace and key_config directly.
       # @param topic [String]
-      # @param config_class [Class < Deimos::Consumer|Deimos::Producer>]
+      # @param config_class [Class<Deimos::Consumer>,Class<Deimos::Producer>]
       # @param schema [String]
       # @param namespace [String]
       # @param key_config [Hash]
-      # @param num_messages [Number]
+      # @param num_messages [Integer]
       # @return [Array<Hash>]
       def self.get_messages_for(topic:, schema: nil, namespace: nil, key_config: nil,
                                 config_class: nil, num_messages: 10)
@@ -106,6 +113,7 @@ module Deimos
       # @param frk_consumer [Class]
       # @param num_messages [Integer] If this number is >= the number
       #   of messages in the topic, all messages will be consumed.
+      # @return [void]
       def self.consume(topic:, frk_consumer:, num_messages: 10)
         listener = SeekListener.new(
           handler: frk_consumer,

data/lib/deimos/utils/lag_reporter.rb

@@ -24,6 +24,7 @@ module Deimos
 
         # @param topic [String]
         # @param partition [Integer]
+        # @return [void]
         def report_lag(topic, partition)
           self.topics[topic.to_s] ||= Topic.new(topic, self)
           self.topics[topic.to_s].report_lag(partition)
@@ -32,6 +33,7 @@ module Deimos
         # @param topic [String]
         # @param partition [Integer]
         # @param offset [Integer]
+        # @return [void]
         def assign_current_offset(topic, partition, offset)
           self.topics[topic.to_s] ||= Topic.new(topic, self)
           self.topics[topic.to_s].assign_current_offset(partition, offset)
@@ -56,11 +58,15 @@ module Deimos
         end
 
         # @param partition [Integer]
+        # @param offset [Integer]
+        # @return [void]
         def assign_current_offset(partition, offset)
           self.partition_current_offsets[partition.to_i] = offset
         end
 
         # @param partition [Integer]
+        # @param offset [Integer]
+        # @return [Integer]
         def compute_lag(partition, offset)
           begin
             client = Phobos.create_kafka_client
@@ -74,6 +80,7 @@ module Deimos
         end
 
         # @param partition [Integer]
+        # @return [void]
         def report_lag(partition)
           current_offset = self.partition_current_offsets[partition.to_i]
           return unless current_offset
@@ -94,6 +101,7 @@ module Deimos
 
       class << self
         # Reset all group information.
+        # @return [void]
         def reset
           @groups = {}
         end
@@ -103,6 +111,7 @@ module Deimos
         # topic = event.payload.fetch(:topic)
         # partition = event.payload.fetch(:partition)
         # @param payload [Hash]
+        # @return [void]
         def message_processed(payload)
           offset = payload[:offset] || payload[:last_offset]
           topic = payload[:topic]
@@ -116,6 +125,7 @@ module Deimos
         end
 
         # @param payload [Hash]
+        # @return [void]
         def offset_seek(payload)
           offset = payload[:offset]
           topic = payload[:topic]
@@ -129,6 +139,7 @@ module Deimos
         end
 
         # @param payload [Hash]
+        # @return [void]
         def heartbeat(payload)
           group = payload[:group_id]
           synchronize do

data/lib/deimos/utils/schema_controller_mixin.rb

@@ -31,6 +31,7 @@ module Deimos
         # @param kwactions [String]
         # @param request [String]
         # @param response [String]
+        # @return [void]
         def schemas(*actions, request: nil, response: nil, **kwactions)
           actions.each do |action|
             request ||= action.to_s.titleize
@@ -49,6 +50,7 @@ module Deimos
 
         # Set the namespace for both requests and responses.
         # @param name [String]
+        # @return [void]
         def namespace(name)
           request_namespace(name)
           response_namespace(name)
@@ -56,12 +58,14 @@ module Deimos
 
         # Set the namespace for requests.
         # @param name [String]
+        # @return [void]
         def request_namespace(name)
           namespaces[:request] = name
         end
 
         # Set the namespace for repsonses.
         # @param name [String]
+        # @return [void]
         def response_namespace(name)
           namespaces[:response] = name
         end
@@ -94,6 +98,7 @@ module Deimos
       end
 
       # Decode the payload with the parameters.
+      # @return [void]
       def decode_schema
         namespace, schema = parse_namespace(:request)
         decoder = Deimos.schema_backend(schema: schema, namespace: namespace)
@@ -109,6 +114,9 @@ module Deimos
 
       # Render a hash into a payload as specified by the configured schema and namespace.
       # @param payload [Hash]
+      # @param schema [String]
+      # @param namespace [String]
+      # @return [void]
       def render_schema(payload, schema: nil, namespace: nil)
         namespace, schema = parse_namespace(:response) if !schema && !namespace
         encoder = Deimos.schema_backend(schema: schema, namespace: namespace)

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.16.1'
+  VERSION = '1.16.4'
 end

data/lib/deimos.rb

@@ -45,7 +45,7 @@ require 'erb'
 # Parent module.
 module Deimos
   class << self
-    # @return [Class < Deimos::SchemaBackends::Base]
+    # @return [Class<Deimos::SchemaBackends::Base>]
     def schema_backend_class
       backend = Deimos.config.schema.backend.to_s
 
@@ -54,7 +54,7 @@ module Deimos
       "Deimos::SchemaBackends::#{backend.classify}".constantize
     end
 
-    # @param schema [String|Symbol]
+    # @param schema [String, Symbol]
     # @param namespace [String]
     # @return [Deimos::SchemaBackends::Base]
     def schema_backend(schema:, namespace:)
@@ -81,6 +81,7 @@ module Deimos
 
     # Start the DB producers to send Kafka messages.
     # @param thread_count [Integer] the number of threads to start.
+    # @return [void]
     def start_db_backend!(thread_count: 1)
       Sigurd.exit_on_signal = true
       if self.config.producers.backend != :db

data/lib/generators/deimos/active_record_generator.rb

@@ -69,7 +69,7 @@ module Deimos
       end
 
       desc 'Generate migration for a table based on an existing schema.'
-      # :nodoc:
+      # @return [void]
       def generate
         migration_template('migration.rb', "db/migrate/create_#{table_name.underscore}.rb")
         template('model.rb', "app/models/#{table_name.underscore.singularize}.rb")

data/lib/generators/deimos/db_backend_generator.rb

@@ -34,6 +34,7 @@ module Deimos
       end
 
       # Main method to create all the necessary files
+      # @return [void]
       def generate
         if Rails.version < '4'
           migration_template('rails3_migration',

data/lib/generators/deimos/db_poller_generator.rb

@@ -34,6 +34,7 @@ module Deimos
       end
 
       # Main method to create all the necessary files
+      # @return [void]
       def generate
         if Rails.version < '4'
           migration_template('rails3_migration',

data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt

@@ -20,7 +20,7 @@
 <%- if @field_assignments.select{ |h| !h[:is_schema_class] }.any? -%>
     ### Attribute Accessors ###
   <%- @field_assignments.select{ |h| !h[:is_schema_class] }.each do |method_definition| -%>
-    # @param <%= method_definition[:method_argument] %> [<%= method_definition[:deimos_type] %>]
+    # @return [<%= method_definition[:deimos_type] %>]
     attr_accessor :<%= method_definition[:field].name %>
   <%- end -%>
 
@@ -28,14 +28,14 @@
 <%- if @field_assignments.select{ |h| h[:is_schema_class] }.any? -%>
     ### Attribute Writers ###
   <%- @field_assignments.select{ |h| h[:is_schema_class] }.each do |method_definition| -%>
-    # @param <%= method_definition[:method_argument] %> [<%= method_definition[:deimos_type] %>]
+    # @return [<%= method_definition[:deimos_type] %>]
     def <%= method_definition[:field].name %>=(<%= method_definition[:method_argument] %>)
     <%- if method_definition[:field_type] == :array -%>
-      @<%= method_definition[:field].name %> = values.map do |value|
+      @<%= method_definition[:field].name %> = values&.map do |value|
         <%= method_definition[:field_initialization] %>
       end
     <%- elsif method_definition[:field_type] == :map -%>
-      @<%= method_definition[:field].name %> = values.transform_values do |value|
+      @<%= method_definition[:field].name %> = values&.transform_values do |value|
         <%= method_definition[:field_initialization] %>
       end
     <%- else -%>
@@ -62,7 +62,15 @@
     def namespace
       '<%= @current_schema.namespace %>'
     end
-
+<%- if @tombstone_assignment %>
+    def self.tombstone(key)
+      record = self.allocate
+<%- if @tombstone_assignment.present? -%>
+      <%= @tombstone_assignment %>
+<%- end -%>
+      record
+    end
+<%- end %>
     # @override
     def as_json(_opts={})
       {

data/lib/generators/deimos/schema_class_generator.rb

@@ -11,11 +11,17 @@ module Deimos
     # Generator for Schema Classes used for the IDE and consumer/producer interfaces
     class SchemaClassGenerator < Rails::Generators::Base
 
+      # @return [Array<Symbol>]
       SPECIAL_TYPES = %i(record enum).freeze
+      # @return [String]
       INITIALIZE_WHITESPACE = "\n#{' ' * 19}"
+      # @return [Array<String>]
       IGNORE_DEFAULTS = %w(message_id timestamp).freeze
+      # @return [String]
       SCHEMA_CLASS_FILE = 'schema_class.rb'
+      # @return [String]
       SCHEMA_RECORD_PATH = File.expand_path('schema_class/templates/schema_record.rb.tt', __dir__).freeze
+      # @return [String]
       SCHEMA_ENUM_PATH = File.expand_path('schema_class/templates/schema_enum.rb.tt', __dir__).freeze
 
       source_root File.expand_path('schema_class/templates', __dir__)
@@ -41,16 +47,17 @@ module Deimos
         # Deimos Consumer or Producer Configuration object
         # @param schema_name [String]
         # @param namespace [String]
-        # @param key_schema_name [String,nil]
-        def generate_classes(schema_name, namespace, key_schema_name)
+        # @param key_config [Hash,nil]
+        # @return [void]
+        def generate_classes(schema_name, namespace, key_config)
           schema_base = Deimos.schema_backend(schema: schema_name, namespace: namespace)
           schema_base.load_schema
-          if key_schema_name.present?
-            key_schema_base = Deimos.schema_backend(schema: key_schema_name, namespace: namespace)
+          if key_config&.dig(:schema)
+            key_schema_base = Deimos.schema_backend(schema: key_config[:schema], namespace: namespace)
             key_schema_base.load_schema
-            generate_class_from_schema_base(key_schema_base)
+            generate_class_from_schema_base(key_schema_base, key_config: nil)
           end
-          generate_class_from_schema_base(schema_base, key_schema_base: key_schema_base)
+          generate_class_from_schema_base(schema_base, key_config: key_config)
         end
 
         # @param schema [Avro::Schema::NamedSchema]
@@ -83,8 +90,9 @@ module Deimos
         end
 
         # @param schema_base [Deimos::SchemaBackends::Base]
-        # @param key_schema_base[Avro::Schema::NamedSchema]
-        def generate_class_from_schema_base(schema_base, key_schema_base: nil)
+        # @param key_config [Hash,nil]
+        # @return [void]
+        def generate_class_from_schema_base(schema_base, key_config: nil)
           @discovered_schemas = Set.new
           @sub_schema_templates = []
           schemas = collect_all_schemas(schema_base.schema_store.schemas.values)
@@ -93,11 +101,11 @@ module Deimos
           sub_schemas = schemas.reject { |s| s.name == schema_base.schema }.sort_by(&:name)
           if Deimos.config.schema.nest_child_schemas
             @sub_schema_templates = sub_schemas.map do |schema|
-              _generate_class_template_from_schema(schema)
+              _generate_class_template_from_schema(schema, nil)
             end
-            write_file(main_schema, key_schema_base)
+            write_file(main_schema, key_config)
           else
-            write_file(main_schema, key_schema_base)
+            write_file(main_schema, key_config)
             sub_schemas.each do |schema|
               write_file(schema, nil)
             end
@@ -105,9 +113,10 @@ module Deimos
         end
 
         # @param schema [Avro::Schema::NamedSchema]
-        # @param key_schema_base [Avro::Schema::NamedSchema, nil]
-        def write_file(schema, key_schema_base)
-          class_template = _generate_class_template_from_schema(schema, key_schema_base)
+        # @param key_config [Hash,nil]
+        # @return [void]
+        def write_file(schema, key_config)
+          class_template = _generate_class_template_from_schema(schema, key_config)
           @modules = Utils::SchemaClass.modules_for(schema.namespace)
           @main_class_definition = class_template
 
@@ -143,7 +152,7 @@ module Deimos
       end
 
       desc 'Generate a class based on configured consumer and producers.'
-      # :nodoc:
+      # @return [void]
       def generate
         _validate
         Rails.logger.info("Generating schemas from Deimos.config to #{Deimos.config.schema.generated_class_path}")
@@ -154,7 +163,7 @@ module Deimos
           key_schema_name = config.key_config[:schema]
           found_schemas.add("#{namespace}.#{schema_name}")
           found_schemas.add("#{namespace}.#{key_schema_name}") if key_schema_name
-          generate_classes(schema_name, namespace, key_schema_name)
+          generate_classes(schema_name, namespace, config.key_config)
         end
 
         Deimos.config.consumer_objects.each do |config|
@@ -163,7 +172,7 @@ module Deimos
           key_schema_name = config.key_config[:schema]
           found_schemas.add("#{namespace}.#{schema_name}")
           found_schemas.add("#{namespace}.#{key_schema_name}") if key_schema_name
-          generate_classes(schema_name, namespace, key_schema_name)
+          generate_classes(schema_name, namespace, config.key_config)
         end
 
         generate_from_schema_files(found_schemas)
@@ -191,10 +200,10 @@ module Deimos
       end
 
       # @param schema[Avro::Schema::NamedSchema]
-      # @param key_schema_base[Avro::Schema::NamedSchema]
+      # @param key_config[Hash,nil]
       # @return [String]
-      def _generate_class_template_from_schema(schema, key_schema_base=nil)
-        _set_instance_variables(schema, key_schema_base)
+      def _generate_class_template_from_schema(schema, key_config)
+        _set_instance_variables(schema, key_config)
 
         temp = schema.is_a?(Avro::Schema::RecordSchema) ? _record_class_template : _enum_class_template
         res = ERB.new(temp, nil, '-')
@@ -202,20 +211,38 @@ module Deimos
       end
 
       # @param schema[Avro::Schema::NamedSchema]
-      # @param key_schema_base[Avro::Schema::NamedSchema]
-      def _set_instance_variables(schema, key_schema_base=nil)
+      # @param key_config [Hash,nil]
+      def _set_instance_variables(schema, key_config)
         schema_is_record = schema.is_a?(Avro::Schema::RecordSchema)
         @current_schema = schema
         return unless schema_is_record
 
         @fields = fields(schema)
-        if key_schema_base.present?
+        key_schema = nil
+        if key_config&.dig(:schema)
+          key_schema_base = Deimos.schema_backend(schema: key_config[:schema], namespace: schema.namespace)
           key_schema_base.load_schema
           key_schema = key_schema_base.schema_store.schemas.values.first
           @fields << Deimos::SchemaField.new('payload_key', key_schema, [], nil)
         end
         @initialization_definition = _initialization_definition
         @field_assignments = _field_assignments
+        @tombstone_assignment = _tombstone_assignment(key_config, key_schema)
+      end
+
+      def _tombstone_assignment(key_config, key_schema)
+        return nil unless key_config
+
+        if key_config[:plain]
+          "record.tombstone_key = key"
+        elsif key_config[:field]
+          "record.tombstone_key = key\n      record.#{key_config[:field]} = key"
+        elsif key_schema
+          field_base_type = _field_type(key_schema)
+          "record.tombstone_key = #{field_base_type}.initialize_from_value(key)\n      record.payload_key = key"
+        else
+          ''
+        end
       end
 
       # Defines the initialization method for Schema Records with one keyword argument per line
@@ -234,7 +261,7 @@ module Deimos
         "#{result})"
       end
 
-      # @param [SchemaField]
+      # @param field [SchemaField]
       # @return [String]
       def _field_default(field)
         default = field.default