deimos-ruby 1.16.3 → 1.16.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ee238a0bcc822c984f19d5a58d1bea52025407d931654371831b0104fcb5aed
-  data.tar.gz: 5b767073027dae54910c69be504ac2da023d540557ef592417f1f60254f9e3f3
+  metadata.gz: 7412853b7b878c8bcfbfa130f21b5d2c3d46b4d741f1f6adad9edcb358843a4b
+  data.tar.gz: 5672157db0da4ea4a67893d7a792737e858962a2011cd097fc4af23a9bea60f0
 SHA512:
-  metadata.gz: 904066c010866aaa6ca1173bbdf4e4044c578ac28df8a0ebcf846c25c73cd484fbeb15dea827c3b64935a8f7588f4a62b65d1bdf6f28604d0f1e588cdb8cb15a
-  data.tar.gz: 3cec3d2ae05d855fb9835252646b6cdef16034f600b0c1b7ed169fc8b7969bb0d6a9847fd5b35092e29ca910daa91cad6b3d838425a3b577dc484d7937af1339
+  metadata.gz: 444d6e14343916242caaa612d1e1c731ce45423ef41dc5e2cf4d9a85e8d618851001d52cf30ef2c2db060b2631bcaa4daa921ddd66eed93636d5d09ac5ad47e2
+  data.tar.gz: 253428dc65fc2d86108671a9e03dd85e9a9afa97681dd562efe2a6ec7d61317002a375705ccc7e0b267da31041a0d4c5b2fce960b7e73195dfdad651c5e77b09

data/.gitignore

@@ -40,3 +40,4 @@ Gemfile.lock
 test.sqlite3
 
 .idea
+/.gem_rbs_collection/

data/CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 1.16.4 - 2022-09-09
+
+- Now generates RBS types.
+- Use `allocate` instead of `new` in `tombstone` to avoid issues with required fields in `initialize`.
+
 # 1.16.3 - 2022-09-08
 
 - Add the `tombstone` method to schema classes.

data/Gemfile

@@ -4,3 +4,9 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in boilerplate.gemspec
 gemspec
+
+if !ENV['CI'] || ENV['CI'] == ''
+  # TODO: once all PRs are merged, add this to gemspec as a development dependency
+  gem 'sord', git: 'git@github.com:dorner/sord.git', ref: 'local-develop'
+end
+

data/README.md

@@ -1198,6 +1198,11 @@ decoded = Deimos.decode(schema: 'MySchema', namespace: 'com.my-namespace', paylo
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/flipp-oss/deimos .
 
+You can/should re-generate RBS types when methods or classes change by running the following:
+
+    rbs collection install # if you haven't done it recently
+    bundle exec sord --hide-private --no-sord-comments sig/defs.rbs --tags 'override:Override'
+
 ### Linting
 
 Deimos uses Rubocop to lint the code. Please run Rubocop on your code 

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -15,8 +15,9 @@ module Deimos
       # If two messages in a batch have the same key, we cannot process them
       # in the same operation as they would interfere with each other. Thus
       # they are split
-      # @param payloads [Array<Hash|Deimos::SchemaClass::Record>] Decoded payloads
+      # @param payloads [Array<Hash,Deimos::SchemaClass::Record>] Decoded payloads
       # @param metadata [Hash] Information about batch, including keys.
+      # @return [void]
       def consume_batch(payloads, metadata)
         messages = payloads.
           zip(metadata[:keys]).
@@ -59,6 +60,7 @@ module Deimos
       # All messages are split into slices containing only unique keys, and
       # each slice is handles as its own batch.
       # @param messages [Array<Message>] List of messages.
+      # @return [void]
       def uncompacted_update(messages)
         BatchSlicer.
           slice(messages).
@@ -69,6 +71,7 @@ module Deimos
       # All messages with payloads are passed to upsert_records.
       # All tombstones messages are passed to remove_records.
       # @param messages [Array<Message>] List of messages.
+      # @return [void]
       def update_database(messages)
         # Find all upserted records (i.e. that have a payload) and all
         # deleted record (no payload)
@@ -81,6 +84,7 @@ module Deimos
       # Upsert any non-deleted records
       # @param messages [Array<Message>] List of messages for a group of
       # records to either be updated or inserted.
+      # @return [void]
       def upsert_records(messages)
         key_cols = key_columns(messages)
 
@@ -119,6 +123,7 @@ module Deimos
       # Delete any records with a tombstone.
       # @param messages [Array<Message>] List of messages for a group of
       # deleted records.
+      # @return [void]
       def remove_records(messages)
         clause = deleted_query(messages)
 
@@ -128,7 +133,7 @@ module Deimos
       # Create an ActiveRecord relation that matches all of the passed
       # records. Used for bulk deletion.
       # @param records [Array<Message>] List of messages.
-      # @return ActiveRecord::Relation Matching relation.
+      # @return [ActiveRecord::Relation] Matching relation.
       def deleted_query(records)
         keys = records.
           map { |m| record_key(m.key)[@klass.primary_key] }.

data/lib/deimos/active_record_consume/batch_slicer.rb

@@ -11,6 +11,8 @@ module Deimos
       # slices that maintain the correct order.
       # E.g. Given messages A1, A2, B1, C1, C2, C3, they will be sliced as:
       # [[A1, B1, C1], [A2, C2], [C3]]
+      # @param messages [Array<Message>]
+      # @return [Array<Array<Message>>]
       def self.slice(messages)
         ops = messages.group_by(&:key)
 

data/lib/deimos/active_record_consume/message_consumption.rb

@@ -8,8 +8,8 @@ module Deimos
       # Find the record specified by the given payload and key.
       # Default is to use the primary key column and the value of the first
       # field in the key.
-      # @param klass [Class < ActiveRecord::Base]
-      # @param _payload [Hash|Deimos::SchemaClass::Record]
+      # @param klass [Class<ActiveRecord::Base>]
+      # @param _payload [Hash,Deimos::SchemaClass::Record]
       # @param key [Object]
       # @return [ActiveRecord::Base]
       def fetch_record(klass, _payload, key)
@@ -19,14 +19,16 @@ module Deimos
 
       # Assign a key to a new record.
       # @param record [ActiveRecord::Base]
-      # @param _payload [Hash|Deimos::SchemaClass::Record]
+      # @param _payload [Hash,Deimos::SchemaClass::Record]
       # @param key [Object]
+      # @return [void]
       def assign_key(record, _payload, key)
         record[record.class.primary_key] = key
       end
 
-      # @param payload [Hash|Deimos::SchemaClass::Record] Decoded payloads
+      # @param payload [Hash,Deimos::SchemaClass::Record] Decoded payloads
       # @param metadata [Hash] Information about batch, including keys.
+      # @return [void]
       def consume(payload, metadata)
         unless self.process_message?(payload)
           Deimos.config.logger.debug(
@@ -64,6 +66,7 @@ module Deimos
       end
 
       # @param record [ActiveRecord::Base]
+      # @return [void]
       def save_record(record)
         record.created_at ||= Time.zone.now if record.respond_to?(:created_at)
         record.updated_at = Time.zone.now if record.respond_to?(:updated_at)
@@ -73,6 +76,7 @@ module Deimos
       # Destroy a record that received a null payload. Override if you need
       # to do something other than a straight destroy (e.g. mark as archived).
       # @param record [ActiveRecord::Base]
+      # @return [void]
       def destroy_record(record)
         record&.destroy
       end

data/lib/deimos/active_record_consumer.rb

@@ -23,14 +23,16 @@ module Deimos
     include ActiveRecordConsume::BatchConsumption
 
     class << self
-      # param klass [Class < ActiveRecord::Base] the class used to save to the
+      # @param klass [Class<ActiveRecord::Base>] the class used to save to the
       # database.
+      # @return [void]
       def record_class(klass)
         config[:record_class] = klass
       end
 
-      # param val [Boolean] Turn pre-compaction of the batch on or off. If true,
+      # @param val [Boolean] Turn pre-compaction of the batch on or off. If true,
       # only the last message for each unique key in a batch is processed.
+      # @return [void]
       def compacted(val)
         config[:compacted] = val
       end
@@ -50,14 +52,15 @@ module Deimos
 
     # Override this method (with `super`) if you want to add/change the default
     # attributes set to the new/existing record.
-    # @param payload [Hash|Deimos::SchemaClass::Record]
+    # @param payload [Hash,Deimos::SchemaClass::Record]
     # @param _key [String]
+    # @return [Hash]
     def record_attributes(payload, _key=nil)
       @converter.convert(payload)
     end
 
     # Override this message to conditionally save records
-    # @param _payload [Hash|Deimos::SchemaClass::Record] The kafka message
+    # @param _payload [Hash,Deimos::SchemaClass::Record] The kafka message
     # @return [Boolean] if true, record is created/update.
     #   If false, record processing is skipped but message offset is still committed.
     def process_message?(_payload)

data/lib/deimos/active_record_producer.rb

@@ -17,6 +17,7 @@ module Deimos
       # @param refetch [Boolean] if true, and we are given a hash instead of
       # a record object, refetch the record to pass into the `generate_payload`
       # method.
+      # @return [void]
       def record_class(klass, refetch: true)
         config[:record_class] = klass
         config[:refetch_record] = refetch
@@ -24,12 +25,14 @@ module Deimos
 
       # @param record [ActiveRecord::Base]
       # @param force_send [Boolean]
+      # @return [void]
       def send_event(record, force_send: false)
         send_events([record], force_send: force_send)
       end
 
       # @param records [Array<ActiveRecord::Base>]
       # @param force_send [Boolean]
+      # @return [void]
       def send_events(records, force_send: false)
         primary_key = config[:record_class]&.primary_key
         messages = records.map do |record|

data/lib/deimos/backends/base.rb

@@ -5,8 +5,9 @@ module Deimos
     # Abstract class for all publish backends.
     class Base
       class << self
-        # @param producer_class [Class < Deimos::Producer]
+        # @param producer_class [Class<Deimos::Producer>]
         # @param messages [Array<Deimos::Message>]
+        # @return [void]
         def publish(producer_class:, messages:)
           Deimos.config.logger.info(
             message: 'Publishing messages',
@@ -21,8 +22,9 @@ module Deimos
           execute(producer_class: producer_class, messages: messages)
         end
 
-        # @param producer_class [Class < Deimos::Producer]
+        # @param producer_class [Class<Deimos::Producer>]
         # @param messages [Array<Deimos::Message>]
+        # @return [void]
         def execute(producer_class:, messages:)
           raise NotImplementedError
         end

data/lib/deimos/backends/kafka.rb

@@ -7,6 +7,7 @@ module Deimos
       include Phobos::Producer
 
       # Shut down the producer if necessary.
+      # @return [void]
       def self.shutdown_producer
         producer.sync_producer_shutdown if producer.respond_to?(:sync_producer_shutdown)
         producer.kafka_client&.close

data/lib/deimos/backends/kafka_async.rb

@@ -7,6 +7,7 @@ module Deimos
       include Phobos::Producer
 
       # Shut down the producer cleanly.
+      # @return [void]
       def self.shutdown_producer
         producer.async_producer_shutdown
         producer.kafka_client&.close

data/lib/deimos/config/configuration.rb

@@ -32,6 +32,7 @@ module Deimos
   end
 
   # Loads generated classes
+  # @return [void]
   def self.load_generated_schema_classes
     if Deimos.config.schema.generated_class_path.nil?
       raise 'Cannot use schema classes without schema.generated_class_path. Please provide a directory.'
@@ -43,6 +44,7 @@ module Deimos
   end
 
   # Ensure everything is set up correctly for the DB backend.
+  # @!visibility private
   def self.validate_db_backend
     begin
       require 'activerecord-import'
@@ -56,6 +58,7 @@ module Deimos
 
   # Validate that consumers are configured correctly, including their
   # delivery mode.
+  # @!visibility private
   def self.validate_consumers
     Phobos.config.listeners.each do |listener|
       handler_class = listener.handler.constantize
@@ -74,6 +77,7 @@ module Deimos
     end
   end
 
+  # @!visibility private
   # @param kafka_config [FigTree::ConfigStruct]
   def self.configure_producer_or_consumer(kafka_config)
     klass = kafka_config.class_name.constantize

data/lib/deimos/config/phobos_config.rb

@@ -20,7 +20,7 @@ module Deimos
       }.to_h
     end
 
-    # :nodoc:
+    # @return [void]
     def reset!
       super
       Phobos.configure(self.phobos_config)
@@ -115,6 +115,7 @@ module Deimos
     end
 
     # Legacy method to parse Phobos config file
+    # @!visibility private
     def phobos_config_file=(file)
       pconfig = YAML.load(ERB.new(File.read(File.expand_path(file))).result). # rubocop:disable Security/YAMLLoad
         with_indifferent_access

data/lib/deimos/consume/batch_consumption.rb

@@ -9,7 +9,9 @@ module Deimos
       extend ActiveSupport::Concern
       include Phobos::BatchHandler
 
-      # :nodoc:
+      # @param batch [Array<String>]
+      # @param metadata [Hash]
+      # @return [void]
       def around_consume_batch(batch, metadata)
         payloads = []
         _with_span do
@@ -36,12 +38,14 @@ module Deimos
       # Consume a batch of incoming messages.
       # @param _payloads [Array<Phobos::BatchMessage>]
       # @param _metadata [Hash]
+      # @return [void]
       def consume_batch(_payloads, _metadata)
         raise NotImplementedError
       end
 
     protected
 
+      # @!visibility private
       def _received_batch(payloads, metadata)
         Deimos.config.logger.info(
           message: 'Got Kafka batch event',
@@ -70,6 +74,7 @@ module Deimos
         end
       end
 
+      # @!visibility private
       # @param exception [Throwable]
       # @param payloads [Array<Hash>]
       # @param metadata [Hash]
@@ -91,6 +96,7 @@ module Deimos
         _error(exception, payloads, metadata)
       end
 
+      # @!visibility private
       # @param time_taken [Float]
       # @param payloads [Array<Hash>]
       # @param metadata [Hash]
@@ -122,6 +128,7 @@ module Deimos
         )
       end
 
+      # @!visibility private
       # Get payload identifiers (key and message_id if present) for logging.
       # @param payloads [Array<Hash>]
       # @param metadata [Hash]

data/lib/deimos/consume/message_consumption.rb

@@ -8,7 +8,9 @@ module Deimos
       extend ActiveSupport::Concern
       include Phobos::Handler
 
-      # :nodoc:
+      # @param payload [String]
+      # @param metadata [Hash]
+      # @return [void]
       def around_consume(payload, metadata)
         decoded_payload = payload.nil? ? nil : payload.dup
         new_metadata = metadata.dup
@@ -28,6 +30,7 @@ module Deimos
       # Consume incoming messages.
       # @param _payload [String]
       # @param _metadata [Hash]
+      # @return [void]
       def consume(_payload, _metadata)
         raise NotImplementedError
       end

data/lib/deimos/instrumentation.rb

@@ -8,23 +8,29 @@ module Deimos
   # Copied from Phobos instrumentation.
   module Instrumentation
     extend ActiveSupport::Concern
+
+    # @return [String]
     NAMESPACE = 'Deimos'
 
     # :nodoc:
     module ClassMethods
-      # :nodoc:
+      # @param event [String]
+      # @return [void]
       def subscribe(event)
         ActiveSupport::Notifications.subscribe("#{NAMESPACE}.#{event}") do |*args|
           yield(ActiveSupport::Notifications::Event.new(*args)) if block_given?
         end
       end
 
-      # :nodoc:
+      # @param subscriber [ActiveSupport::Subscriber]
+      # @return [void]
       def unsubscribe(subscriber)
         ActiveSupport::Notifications.unsubscribe(subscriber)
       end
 
-      # :nodoc:
+      # @param event [String]
+      # @param extra [Hash]
+      # @return [void]
       def instrument(event, extra={})
         ActiveSupport::Notifications.instrument("#{NAMESPACE}.#{event}", extra) do |extra2|
           yield(extra2) if block_given?
@@ -39,7 +45,8 @@ module Deimos
   module KafkaListener
     # Listens for any exceptions that happen during publishing and re-publishes
     # as a Deimos event.
-    # @param event [ActiveSupport::Notification]
+    # @param event [ActiveSupport::Notifications::Event]
+    # @return [void]
     def self.send_produce_error(event)
       exception = event.payload[:exception_object]
       return if !exception || !exception.respond_to?(:failed_messages)

data/lib/deimos/kafka_message.rb

@@ -10,6 +10,7 @@ module Deimos
     # Ensure it gets turned into a string, e.g. for testing purposes. It
     # should already be a string.
     # @param mess [Object]
+    # @return [void]
     def message=(mess)
       write_attribute(:message, mess ? mess.to_s : nil)
     end

data/lib/deimos/kafka_source.rb

@@ -6,6 +6,7 @@ module Deimos
   module KafkaSource
     extend ActiveSupport::Concern
 
+    # @return [String]
     DEPRECATION_WARNING = 'The kafka_producer interface will be deprecated ' \
         'in future releases. Please use kafka_producers instead.'
 
@@ -16,6 +17,7 @@ module Deimos
     end
 
     # Send the newly created model to Kafka.
+    # @return [void]
     def send_kafka_event_on_create
       return unless self.persisted?
       return unless self.class.kafka_config[:create]
@@ -24,6 +26,7 @@ module Deimos
     end
 
     # Send the newly updated model to Kafka.
+    # @return [void]
     def send_kafka_event_on_update
       return unless self.class.kafka_config[:update]
 
@@ -41,6 +44,7 @@ module Deimos
     end
 
     # Send a deletion (null payload) event to Kafka.
+    # @return [void]
     def send_kafka_event_on_destroy
       return unless self.class.kafka_config[:delete]
 
@@ -80,6 +84,7 @@ module Deimos
       # the inputs (arrays, hashes, records etc.)
       # Basically we want to first do the import, then reload the records
       # and send them to Kafka.
+      # @!visibility private
       def import_without_validations_or_callbacks(column_names,
                                                   array_of_attributes,
                                                   options={})

data/lib/deimos/kafka_topic_info.rb

@@ -50,6 +50,7 @@ module Deimos
       # moves on to the next one.
       # @param topic [String]
       # @param lock_id [String]
+      # @return [void]
       def clear_lock(topic, lock_id)
         self.where(topic: topic, locked_by: lock_id).
           update_all(locked_by: nil,
@@ -66,6 +67,7 @@ module Deimos
       # was in a good state.
       # @param except_topics [Array<String>] the list of topics we've just
       # realized had messages in them, meaning all other topics were empty.
+      # @return [void]
       def ping_empty_topics(except_topics)
         records = KafkaTopicInfo.where(locked_by: nil).
           where('topic not in(?)', except_topics)
@@ -79,6 +81,7 @@ module Deimos
       # and allows the caller to continue to the next topic.
       # @param topic [String]
       # @param lock_id [String]
+      # @return [void]
       def register_error(topic, lock_id)
         record = self.where(topic: topic, locked_by: lock_id).last
         attr_hash = { locked_by: nil,
@@ -93,6 +96,7 @@ module Deimos
       # working on those messages and to continue.
       # @param topic [String]
       # @param lock_id [String]
+      # @return [void]
       def heartbeat(topic, lock_id)
         self.where(topic: topic, locked_by: lock_id).
           update_all(locked_at: Time.zone.now)

data/lib/deimos/message.rb

@@ -3,11 +3,26 @@
 module Deimos
   # Basically a struct to hold the message as it's processed.
   class Message
-    attr_accessor :payload, :key, :partition_key, :encoded_key,
-                  :encoded_payload, :topic, :producer_name
+    # @return [Hash]
+    attr_accessor :payload
+    # @return [Hash, String, Integer]
+    attr_accessor :key
+    # @return [Integer]
+    attr_accessor :partition_key
+    # @return [String]
+    attr_accessor  :encoded_key
+    # @return [String]
+    attr_accessor  :encoded_payload
+    # @return [String]
+    attr_accessor  :topic
+    # @return [String]
+    attr_accessor  :producer_name
 
     # @param payload [Hash]
     # @param producer [Class]
+    # @param topic [String]
+    # @param key [String, Integer, Hash]
+    # @param partition_key [Integer]
     def initialize(payload, producer, topic: nil, key: nil, partition_key: nil)
       @payload = payload&.with_indifferent_access
       @producer_name = producer&.name
@@ -19,6 +34,7 @@ module Deimos
     # Add message_id and timestamp default values if they are in the
     # schema and don't already have values.
     # @param fields [Array<String>] existing name fields in the schema.
+    # @return [void]
     def add_fields(fields)
       return if @payload.except(:payload_key, :partition_key).blank?
 
@@ -31,6 +47,7 @@ module Deimos
     end
 
     # @param encoder [Deimos::SchemaBackends::Base]
+    # @return [void]
     def coerce_fields(encoder)
       return if payload.nil?
 

data/lib/deimos/metrics/datadog.rb

@@ -6,7 +6,8 @@ module Deimos
   module Metrics
     # A Metrics wrapper class for Datadog.
     class Datadog < Metrics::Provider
-      # :nodoc:
+      # @param config [Hash]
+      # @param logger [Logger]
       def initialize(config, logger)
         raise 'Metrics config must specify host_ip' if config[:host_ip].nil?
         raise 'Metrics config must specify host_port' if config[:host_port].nil?

data/lib/deimos/metrics/mock.rb

@@ -5,8 +5,8 @@ require 'deimos/metrics/provider'
 module Deimos
   module Metrics
     # A mock Metrics wrapper which just logs the metrics
-    class Mock
-      # :nodoc:
+    class Mock < Provider
+      # @param logger [Logger,nil]
       def initialize(logger=nil)
         @logger = logger || Logger.new(STDOUT)
         @logger.info('MockMetricsProvider initialized')

data/lib/deimos/metrics/provider.rb

@@ -7,20 +7,25 @@ module Deimos
       # Send an counter increment metric
       # @param metric_name [String] The name of the counter metric
       # @param options [Hash] Any additional options, e.g. :tags
+      # @return [void]
       def increment(metric_name, options={})
         raise NotImplementedError
       end
 
       # Send an counter increment metric
       # @param metric_name [String] The name of the counter metric
+      # @param count [Integer]
       # @param options [Hash] Any additional options, e.g. :tags
+      # @return [void]
       def gauge(metric_name, count, options={})
         raise NotImplementedError
       end
 
       # Send an counter increment metric
       # @param metric_name [String] The name of the counter metric
+      # @param count [Integer]
       # @param options [Hash] Any additional options, e.g. :tags
+      # @return [void]
       def histogram(metric_name, count, options={})
         raise NotImplementedError
       end
@@ -28,6 +33,7 @@ module Deimos
       # Time a yielded block, and send a timer metric
       # @param metric_name [String] The name of the metric
       # @param options [Hash] Any additional options, e.g. :tags
+      # @return [void]
       def time(metric_name, options={})
         raise NotImplementedError
       end

data/lib/deimos/monkey_patches/phobos_cli.rb

@@ -2,7 +2,7 @@
 
 require 'phobos/cli/start'
 
-# :nodoc:
+#@!visibility private
 module Phobos
   # :nodoc:
   module CLI

data/lib/deimos/monkey_patches/phobos_producer.rb

@@ -2,6 +2,7 @@
 
 require 'phobos/producer'
 
+#@!visibility private
 module Phobos
   module Producer
     # :nodoc: