flipp-ruby-kafka-temp-fork 0.0.1

data/docs/UPGRADE_1.0.md

@@ -0,0 +1,39 @@
+# Upgrading to v2.0
+
+The following breaking changes were introduced in version 1.0:
+
+* Lag reporting is now optional, and controlled by `config.report_lag`. To
+  re-enable it, you must set `config.report_lag` to true. We recommend you
+  only set this to true on production or staging environments and not
+  development or test.
+* Topics that have no keys now must use `key_config none: true` instead
+  of `key_config plain: true`.
+* `KafkaPublishFailure` has been removed - any code that relies on it must
+  likewise be removed, and the database table can also be dropped. You can
+  also remove any reference to the `payloads_for_resend` method defined
+  on your producers.
+    * Its replacement is the new `db` [publish backend](../README.md#database-backend).
+      Note that to use this option, you need to have [activerecord-import](https://github.com/zdennis/activerecord-import)
+      in your Gemfile.
+* `KafkaSource` will now send messages inside a transaction rather than
+  after a commit. Any code (e.g. RSpecs) that relies on the previous 
+  behavior must be modified.
+* `TestHelpers` no longer automatically stub producers and consumers, to aid
+  in the integration testing scenario. Instead, you should add a `before(:each)`
+  hook that calls `stub_producers_and_consumers!` Along with this, the
+  `kafka_integration` metadata key that used to check for this behavior
+  is no longer supported or needed, and can be removed from any tests that
+  use it. This allows for more flexibility and explicitness 
+  in how to determine if tests are integration tests.
+* The `PlatformSchemaValidation` class has moved to the `Utils` module, so
+  it is now `FlippRubyKafka::Utils::PlatformSchemaValidation`.
+
+Additional changes introduced in v2.0 (which uses Deimos under the hood):
+
+* Removed the `sync_produce` option as it's now redundant with the 
+  `publish_backend` option. `publish_backend` can be set to either
+  `:kafka` or `:kafka_async` to reproduce this option.
+* Changed the name of `rake phobos:start` to `rake deimos:start`.
+* Configuration is now mostly handled by flipp-ruby-kafka, including
+  defaults for the MS platform. See README for how configuration should
+  look now.

data/flipp-ruby-kafka.gemspec

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'flipp_ruby_kafka/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'flipp-ruby-kafka-temp-fork'
+  spec.version       = FlippRubyKafka::VERSION
+  spec.authors       = ['Daniel Orner']
+  spec.email         = ['daniel.orner@wishabi.com']
+  spec.summary       = 'Flipp Kafka libraries for Ruby.'
+  spec.description   = 'Write a longer description. Optional.'
+  spec.homepage      = ''
+
+  spec.files         = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency('ddtrace', '>= 0.11')
+  spec.add_runtime_dependency('deimos-temp-fork', '>= 0.0.2')
+  spec.add_runtime_dependency('flipp-ruby-dogstatsd', '~> 1.0')
+  spec.add_runtime_dependency('railties', '>= 3')
+
+  spec.add_development_dependency('flipp_ruby_style')
+  spec.add_development_dependency('guard', '~> 2')
+  spec.add_development_dependency('guard-rspec', '~> 4')
+  spec.add_development_dependency('guard-rubocop', '~> 1')
+  spec.add_development_dependency('rspec', '~> 3')
+  spec.add_development_dependency('rspec_junit_formatter', '~>0.3')
+  spec.add_development_dependency('rubocop', '~> 0.58')
+  spec.add_development_dependency('rubocop-rspec', '~> 1.27')
+end

data/kafkateria/factories/my_schema.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :myschema, parent: :base do
+    topic { 'my_consume_topic' }
+    schema { 'MySchema' }
+    namespace { 'com.my-namespace' }
+    key_config { { plain: true } }
+    fields do
+      {
+        test_id: -> { Faker::Name.name },
+        some_int: -> { Faker::Number.number(5) }
+      }
+    end
+  end
+end

data/kafkateria/factories/my_schema_with_id.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :myschemawithid, parent: :base do
+    topic { 'my_consume_topic_with_id' }
+    schema { 'MySchemaWithId' }
+    namespace { 'com.my-namespace' }
+    key_config { { schema: 'MySchema-key' } }
+    fields do
+      {
+        test_id: -> { Faker::Name.name },
+        some_int: -> { Faker::Number.number(5) }
+      }
+    end
+  end
+end

data/lib/flipp_ruby_kafka.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require 'deimos'
+require 'deimos/tracing/datadog'
+require 'flipp-ruby-dogstatsd'
+require 'rails'
+
+# alias all the Deimos stuff
+FlippRubyKafka = Deimos
+
+require 'flipp_ruby_kafka/version'
+require 'flipp_ruby_kafka/utils/platform_schema_validation'
+require 'flipp_ruby_kafka/utils/platform_topic_validation'
+require 'flipp_ruby_kafka/flipp_datadog_metrics'
+require 'generators/flipp_ruby_kafka/topic_validation/topic_validation_generator'
+require 'generators/flipp_ruby_kafka/schema_validation/schema_validation_generator'
+
+# Parent module.
+module FlippRubyKafka
+  after_configure do
+    formatter = self.config.logger.formatter
+    phobos_formatter = self.config.phobos_logger.formatter.clone
+    kafka_formatter = self.config.kafka.logger.formatter.clone
+
+    if formatter&.respond_to?(:push_global_tags) # FlippRailsLogger
+      self.config.phobos_logger.formatter = phobos_formatter
+      self.config.kafka.logger.formatter = kafka_formatter
+      formatter.push_global_tags('Deimos') unless formatter.current_tags.include?('Deimos')
+    end
+    if phobos_formatter&.respond_to?(:push_global_tags)
+      phobos_formatter.push_global_tags('Phobos') unless phobos_formatter.current_tags.include?('Phobos')
+    end
+    if kafka_formatter&.respond_to?(:push_global_tags)
+      kafka_formatter.push_global_tags('RubyKafka') unless kafka_formatter.current_tags.include?('RubyKafka')
+    end
+  end
+
+  define_settings do
+    setting :logger, default_proc: proc { Rails.logger.clone }
+    setting :schema do
+      setting :backend, default_proc: proc { Rails.env.development? ? :avro_validation : :avro_schema_registry }
+      setting :path, default_proc: proc { "#{Rails.root}/app/schemas" }
+      setting :registry_url, ENV['KAFKA_SCHEMA_REGISTRY'] || 'http://127.0.0.1:8081'
+    end
+    setting :consumers do
+      setting :report_lag, default_proc: proc { !%w(test development).include?(Rails.env) }
+      setting(:fatal_error, proc { defined?(ActiveRecord) && !ActiveRecord::Base.connection.active? })
+    end
+    setting :kafka do
+      setting :seed_brokers, ENV['KAFKA_SSL_BROKERS'] || '127.0.0.1:9092'
+      setting :ssl do
+        setting :enabled, default_proc: proc { !%w(test development).include?(Rails.env) }
+        setting :ca_cert, '/var/certs/kafka-broker-ca-public-cert'
+        setting :client_cert, '/var/certs/kafka-client-public-key'
+        setting :client_cert_key, '/var/certs/kafka-client-private-key-decrypted'
+        setting :verify_hostname, false
+      end
+    end
+
+    # Base URL for Kafkateria - typically something like localhost:3000
+    # for local testing or kafkateria:3000 for docker-compose.
+    setting :kafkateria_url, 'http://127.0.0.1:3000'
+  end
+
+  class << self
+    # Configure datadog. Must be called after FlippDatadog is configured.
+    def configure_datadog
+      service_name = "#{FlippDatadog.config&.service || 'frk'}-consumer"
+      self.config.metrics = FlippRubyKafka::FlippDatadogMetrics.new(nil, self.config.logger)
+      self.config.tracer = Deimos::Tracing::Datadog.new(service_name: service_name)
+    end
+  end
+end

data/lib/flipp_ruby_kafka/flipp_datadog_metrics.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'flipp-ruby-dogstatsd'
+require 'deimos/metrics/datadog'
+
+module FlippRubyKafka
+  # Datadog metrics provider that uses the FlippDatadog gem.
+  # Configuration must happen before FlippRubyKafka is configured.
+  class FlippDatadogMetrics < Deimos::Metrics::Datadog
+    # :nodoc:
+    def initialize(_config, _logger)
+      @client = FlippDatadog.client
+    end
+  end
+end

data/lib/flipp_ruby_kafka/test_helpers.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require 'deimos/test_helpers'
+FlippRubyKafka = Deimos # in case this is being included directly
+
+module FlippRubyKafka
+  # :nodoc:
+  module TestHelpers
+    # Use Kafkateria to clear a topic.
+    # @param topic [String]
+    def kafkateria_delete_topic(topic)
+      base = FlippRubyKafka.config.kafkateria_url
+      uri = URI("#{base}/v1/topics/delete_topic?topic=#{topic}")
+      request = Net::HTTP::Post.new(uri)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.request(request)
+    end
+
+    # Use Kafkateria to generate and produce messages.
+    # @param topic [String]
+    # @param num_messages [Integer]
+    # @param traits [Array<Symbol>]
+    # @param values [Hash]
+    def kafkateria_produce_messages(topic:, num_messages: 1, traits: [], trait_params: {}, values: {}, null_payload: false, key: nil)
+      base = FlippRubyKafka.config.kafkateria_url
+      uri = URI("#{base}//v1/messages/generate_and_publish")
+      req = Net::HTTP::Post.new(uri, 'Content-Type' => 'application/json')
+      req.body = { topic: topic, values: values,
+                   traits: traits, trait_params: trait_params, num_messages: num_messages, null_payload: null_payload, key: key }.to_json
+      response = Net::HTTP.start(uri.hostname, uri.port) do |http|
+        http.request(req)
+      end
+      if response.code.to_s != '200'
+        raise "Failed #{response.code} producing messages to #{topic}: #{response.body}"
+      end
+
+      JSON.parse(response.body || [])
+    end
+
+    # @param topic [String]
+    # @param num_messages [Integer]
+    # @return [Array<Hash>] with :key and :payload keys
+    def kafkateria_get_messages(topic:, num_messages: 10)
+      base = FlippRubyKafka.config.kafkateria_url
+      uri = URI("#{base}//v1/messages?topic=#{topic}&num_messages=#{num_messages}")
+      res = Net::HTTP.get_response(uri)
+      if res.code.to_s != '200'
+        raise "Failed #{res.code} getting messages from #{topic}: #{res.body}"
+      end
+
+      response = JSON.parse(res.body)
+      if response['payloads'].present?
+        response['payloads'].map(&:with_indifferent_access)
+      else
+        response
+      end
+    end
+  end
+end

data/lib/flipp_ruby_kafka/utils/platform_schema_validation.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module FlippRubyKafka
+  module Utils
+    # Generates the Microservice Platform schema validation file.
+    # Run this before committing schema changes (or include it in your CircleCI
+    # config if you don't mind having to build the same image multiple times).
+    # Run this using `rails runner` or in Rails console if you're using Rails.
+    class PlatformSchemaValidation
+      # @param base_path [String] Set this to Rails.root if running Rails.
+      def initialize(base_path: '')
+        @base_path ||= FlippRubyKafka.config.schema.path.gsub(%r{^#{base_path}\/}, '')
+      end
+
+      # @param klass [Class < FlippRubyKafka::Producer]
+      # @return [Hash]
+      def process_producer(klass)
+        schema = klass.config[:schema]
+        topic = klass.config[:topic]
+        return if topic.blank?
+
+        namespace = klass.config[:namespace] || FlippRubyKafka.config.producers.schema_namespace
+        path = namespace.tr('.', '/')
+        {
+          'file' => File.join(@base_path, path, "#{schema}.avsc"),
+          'subject' => "#{topic}-value"
+        }
+      end
+
+      # @param hash [Hash]
+      # @return [Hash]
+      def process_consumer(hash)
+        klass = hash['handler'].constantize
+        topic = hash['topic']
+        schema = klass.config[:schema]
+        return if schema.blank?
+
+        namespace = klass.config[:namespace] || ''
+        path = namespace.tr('.', '/')
+        {
+          'file' => File.join(@base_path, path, "#{schema}.avsc"),
+          'subject' => "#{topic}-value"
+        }
+      end
+
+      # @return [String]
+      def generate_config
+        @full_config = []
+        FlippRubyKafka::Producer.descendants.each do |klass|
+          next if klass.topic.blank?
+
+          config = process_producer(klass)
+          @full_config << config if config
+        end
+
+        Phobos.config['listeners'].each do |h|
+          config = process_consumer(h)
+          @full_config << config if config
+        end
+
+        {
+          environments: {
+            default: @full_config
+          }
+        }
+      end
+
+      # :nodoc:
+      def run
+        full_json = generate_config
+        File.write('schema-validation.json', JSON.pretty_generate(full_json))
+      end
+    end
+  end
+end

data/lib/flipp_ruby_kafka/utils/platform_topic_validation.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module FlippRubyKafka
+  module Utils
+    # Generates the Microservice Platform topic validation file.
+    # Use rake flipp_ruby_kafka:topic_validation my_service to run it from command line.
+    class PlatformTopicValidation
+      class << self
+        # @param service_name [String]
+        def save_config_to_file(service_name)
+          config = generate_config(service_name)
+          File.open("#{Rails.root}/topic-validation.json", 'w') do |f|
+            f.write(config.to_json)
+          end
+        end
+
+        # @param service_name [String]
+        # @return [Hash]
+        def generate_config(service_name)
+          acls = []
+          Deimos.config.producer_objects.each do |k|
+            next if k.topic.blank?
+
+            acls.push(topic: k.topic, role: 'producer')
+          end
+          Deimos.config.consumer_objects.each do |k|
+            next if k.topic.blank?
+
+            acls.push(topic: k.topic, role: 'consumer')
+          end
+          {
+            service: service_name,
+            acls: acls
+          }
+        end
+      end
+    end
+  end
+end

data/lib/flipp_ruby_kafka/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module FlippRubyKafka
+  VERSION = '0.0.1'
+end

data/lib/generators/flipp_ruby_kafka/schema_validation/schema_validation_generator.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+require 'flipp_ruby_kafka/utils/platform_schema_validation'
+
+module FlippRubyKafka
+  module Generators
+    # Generate the database backend migration.
+    class SchemaValidationGenerator < Rails::Generators::Base
+      desc 'Generate schema validation config file'
+
+      # :nodoc:
+      def self.namespace
+        'flipp_ruby_kafka:schema_validation'
+      end
+
+      # Main method to create all the necessary files
+      def generate
+        FlippRubyKafka::Utils::PlatformSchemaValidation.new(base_path: Rails.root).run
+      end
+    end
+  end
+end

data/lib/generators/flipp_ruby_kafka/topic_validation/topic_validation_generator.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+
+module FlippRubyKafka
+  module Generators
+    # Generate the database backend migration.
+    class TopicValidationGenerator < Rails::Generators::NamedBase
+      desc 'Generate topic validation config file'
+
+      # :nodoc:
+      def self.namespace
+        'flipp_ruby_kafka:topic_validation'
+      end
+
+      # Main method to create all the necessary files
+      def generate
+        FlippRubyKafka::Utils::PlatformTopicValidation.save_config_to_file(name)
+      end
+    end
+  end
+end

data/spec/flipp_ruby_kafka_spec.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+describe FlippRubyKafka do
+
+  it 'should have a version number' do
+    expect(FlippRubyKafka::VERSION).not_to be_nil
+  end
+end