deimos-kafka 1.0.0.pre.beta15

data/lib/deimos/active_record_consumer.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'deimos/consumer'
+
+module Deimos
+  # Consumer that automatically saves the payload into the database.
+  class ActiveRecordConsumer < Consumer
+    class << self
+      # param klass [Class < ActiveRecord::Base] the class used to save to the
+      # database.
+      def record_class(klass)
+        config[:record_class] = klass
+      end
+    end
+
+    # :nodoc:
+    def consume(payload, metadata)
+      key = metadata.with_indifferent_access[:key]
+      klass = self.class.config[:record_class]
+      record = klass.where(klass.primary_key => key).first
+      if payload.nil?
+        destroy_record(record)
+        return
+      end
+      record ||= klass.new
+      attrs = record_attributes(payload.with_indifferent_access)
+      # don't use attributes= - bypass Rails < 5 attr_protected
+      attrs.each do |k, v|
+        record.send("#{k}=", v)
+      end
+      record[klass.primary_key] = key
+      record.created_at ||= Time.zone.now if record.respond_to?(:created_at)
+      record.updated_at ||= Time.zone.now if record.respond_to?(:updated_at)
+      record.save!
+    end
+
+    # Destroy a record that received a null payload. Override if you need
+    # to do something other than a straight destroy (e.g. mark as archived).
+    # @param record [ActiveRecord::Base]
+    def destroy_record(record)
+      record&.destroy
+    end
+
+    # Override this method (with `super`) if you want to add/change the default
+    # attributes set to the new/existing record.
+    # @param payload [Hash]
+    def record_attributes(payload)
+      klass = self.class.config[:record_class]
+      attributes = {}
+      schema = self.class.decoder.avro_schema
+      schema.fields.each do |field|
+        column = klass.columns.find { |c| c.name == field.name }
+        next if column.nil?
+        next if %w(updated_at created_at).include?(field.name)
+
+        attributes[field.name] = _coerce_field(field, column, payload[field.name])
+      end
+      attributes
+    end
+
+  private
+
+    # @param field [Avro::Schema]
+    # @param column [ActiveRecord::ConnectionAdapters::Column]
+    # @param val [Object]
+    def _coerce_field(field, column, val)
+      return nil if val.nil?
+
+      field_type = field.type.type.to_sym
+      if field_type == :union
+        union_types = field.type.schemas.map { |s| s.type.to_sym }
+        field_type = union_types.find { |t| t != :null }
+      end
+      if column.type == :datetime && %i(int long).include?(field_type)
+        return Time.zone.strptime(val.to_s, '%s')
+      end
+
+      val
+    end
+  end
+end

data/lib/deimos/active_record_producer.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require 'deimos/producer'
+
+module Deimos
+  # Class which automatically produces a record when given an ActiveRecord
+  # instance or a list of them. Just call `send_events` on a list of records
+  # and they will be auto-published. You can override `generate_payload`
+  # to make changes to the payload before it's published.
+  #
+  # You can also call this with a list of hashes representing attributes.
+  # This is common when using activerecord-import.
+  class ActiveRecordProducer < Producer
+    class << self
+      # Indicate the class this producer is working on.
+      # @param klass [Class]
+      # @param refetch [Boolean] if true, and we are given a hash instead of
+      # a record object, refetch the record to pass into the `generate_payload`
+      # method.
+      def record_class(klass, refetch: true)
+        config[:record_class] = klass
+        config[:refetch_record] = refetch
+      end
+
+      # @param record [ActiveRecord::Base]
+      # @param force_send [Boolean]
+      def send_event(record, force_send: false)
+        send_events([record], force_send: force_send)
+      end
+
+      # @param records [Array<ActiveRecord::Base>]
+      # @param force_send [Boolean]
+      def send_events(records, force_send: false)
+        primary_key = config[:record_class]&.primary_key
+        messages = records.map do |record|
+          if record.respond_to?(:attributes)
+            attrs = record.attributes.with_indifferent_access
+          else
+            attrs = record.with_indifferent_access
+            if config[:refetch_record] && attrs[primary_key]
+              record = config[:record_class].find(attrs[primary_key])
+            end
+          end
+          generate_payload(attrs, record).with_indifferent_access
+        end
+        self.publish_list(messages, force_send: force_send)
+      end
+
+      # Generate the payload, given a list of attributes or a record..
+      # Can be overridden or added to by subclasses.
+      # @param attributes [Hash]
+      # @param _record [ActiveRecord::Base] May be nil if refetch_record
+      # is not set.
+      # @return [Hash]
+      def generate_payload(attributes, _record)
+        schema = self.encoder.avro_schema
+        payload = attributes.stringify_keys
+        payload.delete_if do |k, _|
+          k.to_sym != :payload_key && !schema.fields.find { |f| f.name == k }
+        end
+      end
+    end
+  end
+end

data/lib/deimos/avro_data_coder.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Deimos
+  # Base class for the encoder / decoder classes.
+  class AvroDataCoder
+    attr_accessor :schema, :namespace, :config, :schema_store
+
+    # @param schema [String]
+    # @param namespace [String]
+    # @param schema_store [AvroTurf::SchemaStore]
+    def initialize(schema:, namespace:, schema_store: nil)
+      @schema = schema
+      @namespace = namespace
+      @schema_store = schema_store ||
+                      AvroTurf::SchemaStore.new(path: Deimos.config.schema_path)
+    end
+
+    # @param schema [String]
+    # @return [Avro::Schema]
+    def avro_schema(schema=nil)
+      schema ||= @schema
+      @schema_store.find(schema, @namespace)
+    end
+
+  private
+
+    # @return [AvroTurf]
+    def avro_turf
+      @avro_turf ||= AvroTurf.new(
+        schemas_path: Deimos.config.schema_path,
+        schema_store: @schema_store
+      )
+      @avro_turf
+    end
+
+    # @return [AvroTurf::Messaging]
+    def avro_turf_messaging
+      @avro_turf_messaging ||= AvroTurf::Messaging.new(
+        schema_store: @schema_store,
+        registry_url: Deimos.config.schema_registry_url,
+        schemas_path: Deimos.config.schema_path,
+        namespace: @namespace
+      )
+    end
+
+    # Generate a key schema from the given value schema and key ID. This
+    # is used when encoding or decoding keys from an existing value schema.
+    # @param key_id [Symbol]
+    # @return [Hash]
+    def _generate_key_schema(key_id)
+      return @key_schema if @key_schema
+
+      value_schema = @schema_store.find(@schema, @namespace)
+      key_field = value_schema.fields.find { |f| f.name == key_id.to_s }
+      name = _key_schema_name(@schema)
+      @key_schema = {
+        'type' => 'record',
+        'name' => name,
+        'namespace' => @namespace,
+        'doc' => "Key for #{@namespace}.#{@schema}",
+        'fields' => [
+          {
+            'name' => key_id,
+            'type' => key_field.type.type_sym.to_s
+          }
+        ]
+      }
+      @schema_store.add_schema(@key_schema)
+      @key_schema
+    end
+
+    # @param value_schema [Hash]
+    # @return [String]
+    def _field_name_from_schema(value_schema)
+      raise "Schema #{@schema} not found!" if value_schema.nil?
+      if value_schema['fields'].nil? || value_schema['fields'].empty?
+        raise "Schema #{@schema} has no fields!"
+      end
+
+      value_schema['fields'][0]['name']
+    end
+
+    # @param schema [String]
+    # @return [String]
+    def _key_schema_name(schema)
+      "#{schema.gsub('-value', '')}_key"
+    end
+  end
+end

data/lib/deimos/avro_data_decoder.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'avro_turf/messaging'
+require 'deimos/avro_data_coder'
+
+module Deimos
+  # Service Object to decode avro messages
+  class AvroDataDecoder < AvroDataCoder
+    # Decode some data.
+    # @param payload [Hash|String]
+    # @param schema [String]
+    # @return [Hash]
+    def decode(payload, schema: nil)
+      schema ||= @schema
+      avro_turf_messaging.decode(payload, schema_name: schema)
+    end
+
+    # Decode against a local schema.
+    # @param payload [Hash]
+    # @param schema [String]
+    # @return [Hash]
+    def decode_local(payload, schema: nil)
+      schema ||= @schema
+      avro_turf.decode(payload, schema_name: schema, namespace: @namespace)
+    end
+
+    # @param payload [String] the encoded key.
+    # @param key_id [String|Symbol]
+    # @return [Object] the decoded key (int/long/string).
+    def decode_key(payload, key_id)
+      key_schema = _generate_key_schema(key_id)
+      field_name = _field_name_from_schema(key_schema)
+      decode(payload, schema: key_schema['name'])[field_name]
+    end
+  end
+end

data/lib/deimos/avro_data_encoder.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'avro_turf/messaging'
+require 'deimos/avro_data_coder'
+
+module Deimos
+  # Service Object to decode Avro messages.
+  class AvroDataEncoder < AvroDataCoder
+    # @param payload [Hash]
+    # @param schema [String]
+    # @return [String]
+    def encode_local(payload, schema: nil)
+      schema ||= @schema
+      Avro::SchemaValidator.validate!(avro_schema(schema), payload,
+                                      recursive: true,
+                                      fail_on_extra_fields: true)
+      avro_turf.encode(payload, schema_name: schema, namespace: @namespace)
+    rescue Avro::IO::AvroTypeError
+      # throw a more detailed error
+      value_schema = @schema_store.find(schema, @namespace)
+      Avro::SchemaValidator.validate!(value_schema, payload)
+    end
+
+    # @param payload [Hash]
+    # @param schema [String]
+    # @param topic [String]
+    # @return [String]
+    def encode(payload, schema: nil, topic: nil)
+      schema ||= @schema
+      Avro::SchemaValidator.validate!(avro_schema(schema), payload,
+                                      recursive: true,
+                                      fail_on_extra_fields: true)
+      avro_turf_messaging.encode(payload, schema_name: schema, subject: topic)
+    rescue Avro::IO::AvroTypeError
+      # throw a more detailed error
+      schema = @schema_store.find(@schema, @namespace)
+      Avro::SchemaValidator.validate!(schema, payload)
+    end
+
+    # @param key_id [Symbol|String]
+    # @param key [Object]
+    # @param topic [String]
+    # @return [String] the encoded key.
+    def encode_key(key_id, key, topic=nil)
+      key_schema = _generate_key_schema(key_id)
+      field_name = _field_name_from_schema(key_schema)
+      payload = { field_name => key }
+      encode(payload, schema: key_schema['name'], topic: topic)
+    end
+  end
+end

data/lib/deimos/backends/db.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'deimos/kafka_message'
+
+module Deimos
+  module Backends
+    # Backend which saves messages to the database instead of immediately
+    # sending them.
+    class Db < Deimos::PublishBackend
+      class << self
+        # :nodoc:
+        def execute(producer_class:, messages:)
+          records = messages.map do |m|
+            message = Deimos::KafkaMessage.new(
+              message: m.encoded_payload.to_s.b,
+              topic: m.topic,
+              partition_key: m.partition_key || m.key
+            )
+            message.key = m.encoded_key.to_s.b unless producer_class.config[:no_keys]
+            message
+          end
+          Deimos::KafkaMessage.import(records)
+        end
+      end
+    end
+  end
+end

data/lib/deimos/backends/kafka.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Deimos
+  module Backends
+    # Default backend to produce to Kafka.
+    class Kafka < Deimos::PublishBackend
+      include Phobos::Producer
+
+      # :nodoc:
+      def self.execute(producer_class:, messages:)
+        Deimos.instrument(
+          'produce',
+          producer: producer_class,
+          topic: producer_class.topic,
+          payloads: messages.map(&:payload)
+        ) do
+          producer.publish_list(messages.map(&:encoded_hash))
+          Deimos.config.metrics&.increment(
+            'publish',
+            tags: %W(status:success topic:#{producer_class.topic}),
+            by: messages.size
+          )
+        end
+      end
+    end
+  end
+end

data/lib/deimos/backends/kafka_async.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Deimos
+  module Backends
+    # Backend which produces to Kafka via an async producer.
+    class KafkaAsync < Deimos::PublishBackend
+      include Phobos::Producer
+
+      # :nodoc:
+      def self.execute(producer_class:, messages:)
+        Deimos.instrument(
+          'produce',
+          producer: producer_class,
+          topic: producer_class.topic,
+          payloads: messages.map(&:payload)
+        ) do
+          producer.async_publish_list(messages.map(&:encoded_hash))
+          Deimos.config.metrics&.increment(
+            'publish',
+            tags: %W(status:success topic:#{producer_class.topic}),
+            by: messages.size
+          )
+        end
+      end
+    end
+  end
+end

data/lib/deimos/configuration.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Deimos
+  # Class to hold configuration.
+  class Configuration
+    # @return [Logger]
+    attr_accessor :logger
+    attr_accessor :phobos_logger
+    attr_accessor :kafka_logger
+
+    # By default, consumer errors will be consumed and logged to
+    # the metrics provider.
+    # Set this to true to force the error to be raised.
+    # @return [Boolean]
+    attr_accessor :reraise_consumer_errors
+
+    # @return [String]
+    attr_accessor :schema_registry_url
+
+    # @return [String]
+    attr_accessor :seed_broker
+
+    # Local path to schemas.
+    # @return [String]
+    attr_accessor :schema_path
+
+    # Default namespace for all producers. Can remain nil. Individual
+    # producers can override.
+    # @return [String]
+    attr_accessor :producer_schema_namespace
+
+    # Add a prefix to all topic names. This can be useful if you're using
+    # the same Kafka broker for different environments that are producing
+    # the same topics.
+    # @return [String]
+    attr_accessor :producer_topic_prefix
+
+    # Disable all actual message producing. Useful when doing things like
+    # mass imports or data space management when events don't need to be
+    # fired.
+    # @return [Boolean]
+    attr_accessor :disable_producers
+
+    # File path to the Phobos configuration file, relative to the application root.
+    # @return [String]
+    attr_accessor :phobos_config_file
+
+    # @return [Boolean]
+    attr_accessor :ssl_enabled
+
+    # @return [String]
+    attr_accessor :ssl_ca_cert
+
+    # @return [String]
+    attr_accessor :ssl_client_cert
+
+    # @return [String]
+    attr_accessor :ssl_client_cert_key
+
+    # Currently can be set to :db, :kafka, or :async_kafka. If using Kafka
+    # directly, set to async in your user-facing app, and sync in your
+    # consumers or delayed workers.
+    # @return [Symbol]
+    attr_accessor :publish_backend
+
+    # @return [Boolean]
+    attr_accessor :report_lag
+
+    # @return [Metrics::Provider]
+    attr_accessor :metrics
+
+    # @return [Tracing::Provider]
+    attr_accessor :tracer
+
+    # :nodoc:
+    def initialize
+      @phobos_config_file = 'config/phobos.yml'
+      @publish_backend = :kafka_async
+    end
+
+    # @param other_config [Configuration]
+    # @return [Boolean]
+    def phobos_config_changed?(other_config)
+      phobos_keys = %w(phobos_config_file ssl_ca_cert ssl_client_cert ssl_client_cert_key)
+      phobos_keys.any? { |key| self.send(key) != other_config.send(key) }
+    end
+  end
+end