deimos-kafka 1.0.0.pre.beta15

data/spec/publish_backend_spec.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+RSpec.describe Deimos::PublishBackend do
+  include_context 'with publish_backend'
+  it 'should call execute' do
+    expect(described_class).to receive(:execute).
+      with(messages: messages, producer_class: MyProducer)
+    described_class.publish(producer_class: MyProducer, messages: messages)
+  end
+end

data/spec/schemas/com/my-namespace/MySchema-key.avsc

@@ -0,0 +1,13 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MySchema-key",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "test_id",
+      "type": "string",
+      "doc": "test string"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/MySchema.avsc

@@ -0,0 +1,18 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MySchema",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "test_id",
+      "type": "string",
+      "doc": "test string"
+    },
+    {
+      "name": "some_int",
+      "type": "int",
+      "doc": "test int"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc

@@ -0,0 +1,18 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MySchemaWithBooleans",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "test_id",
+      "type": "string",
+      "doc": "test string"
+    },
+    {
+      "name": "some_bool",
+      "type": "boolean",
+      "doc": "test bool"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc

@@ -0,0 +1,33 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MySchemaWithDateTimes",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "test_id",
+      "type": "string",
+      "doc": "test string"
+    },
+    {
+      "name": "updated_at",
+      "type": ["int", "null"]
+    },
+    {
+      "name": "some_int",
+      "type": ["null", "int"],
+      "doc": "test int"
+    },
+    {
+      "name": "some_datetime_int",
+      "type": ["null", "int"],
+      "doc": "test datetime"
+    },
+    {
+      "name": "timestamp",
+      "type": "string",
+      "doc": "ISO timestamp",
+      "default": ""
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/MySchemaWithId.avsc

@@ -0,0 +1,28 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MySchemaWithId",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "test_id",
+      "type": "string",
+      "doc": "test string"
+    },
+    {
+      "name": "some_int",
+      "type": "int",
+      "doc": "test int"
+    },
+    {
+      "name": "message_id",
+      "type": "string",
+      "doc": "UUID"
+    },
+    {
+      "name": "timestamp",
+      "type": "string",
+      "doc": "timestamp"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc

@@ -0,0 +1,32 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MySchemaWithUniqueId",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "id",
+      "type": "int"
+    },
+    {
+      "name": "test_id",
+      "type": "string",
+      "doc": "test string"
+    },
+    {
+      "name": "some_int",
+      "type": "int",
+      "doc": "test int"
+    },
+    {
+      "name": "message_id",
+      "type": "string",
+      "doc": "UUID"
+    },
+    {
+      "name": "timestamp",
+      "type": "string",
+      "doc": "timestamp"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/Widget.avsc

@@ -0,0 +1,27 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "Widget",
+  "type": "record",
+  "fields": [
+    {
+      "name": "id",
+      "type": "long"
+    },
+    {
+      "name": "widget_id",
+      "type": "long"
+    },
+    {
+      "name": "name",
+      "type": "string"
+    },
+    {
+      "name": "updated_at",
+      "type": "long"
+    },
+    {
+      "name": "created_at",
+      "type": "long"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/WidgetTheSecond.avsc

@@ -0,0 +1,27 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "WidgetTheSecond",
+  "type": "record",
+  "fields": [
+    {
+      "name": "id",
+      "type": "long"
+    },
+    {
+      "name": "widget_id",
+      "type": "long"
+    },
+    {
+      "name": "model_id",
+      "type": "string"
+    },
+    {
+      "name": "updated_at",
+      "type": "long"
+    },
+    {
+      "name": "created_at",
+      "type": "long"
+    }
+  ]
+}

data/spec/spec_helper.rb

@@ -0,0 +1,207 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
+require 'active_record'
+require 'deimos'
+require 'deimos/metrics/mock'
+require 'deimos/tracing/mock'
+require 'deimos/test_helpers'
+require 'active_support/testing/time_helpers'
+
+# Helpers for Executor/DbProducer
+module TestRunners
+  # Execute a block until it stops failing. This is helpful for testing threads
+  # where we need to wait for them to continue but don't want to rely on
+  # sleeping for X seconds, which is crazy brittle and slow.
+  def wait_for
+    start_time = Time.now
+    begin
+      yield
+    rescue Exception # rubocop:disable Lint/RescueException
+      raise if Time.now - start_time > 2 # 2 seconds is probably plenty of time! <_<
+
+      sleep(0.1)
+      retry
+    end
+  end
+
+  # Test runner
+  class TestRunner
+    attr_accessor :id, :started, :stopped, :should_error
+    # :nodoc:
+    def initialize(id=nil)
+      @id = id
+    end
+
+    # :nodoc:
+    def start
+      if @should_error
+        @should_error = false
+        raise 'OH NOES'
+      end
+      @started = true
+    end
+
+    # :nodoc:
+    def stop
+      @stopped = true
+    end
+  end
+end
+
+# :nodoc:
+module DbConfigs
+  # @param payload [Hash]
+  # @param topic [String]
+  # @param key [String]
+  def build_message(payload, topic, key)
+    message = Deimos::Message.new(payload, Deimos::Producer,
+                                  topic: topic, key: key)
+    message.encoded_payload = message.payload
+    message.encoded_key = message.key
+    message
+  end
+
+  DB_OPTIONS = [
+    {
+      adapter: 'postgresql',
+      port: 5432,
+      username: 'postgres',
+      password: 'root',
+      database: 'postgres',
+      host: ENV['PG_HOST'] || 'localhost'
+    },
+    {
+      adapter: 'mysql2',
+      port: 3306,
+      username: 'root',
+      database: 'test',
+      host: ENV['MYSQL_HOST'] || 'localhost'
+    },
+    {
+      adapter: 'sqlite3',
+      database: 'test.sqlite3'
+    } # this one always needs to be last for non-integration tests
+  ].freeze
+
+  # For each config, run some tests.
+  def each_db_config(subject, &block)
+    DB_OPTIONS.each do |options|
+      describe subject, :integration, db_config: options do
+
+        include_context 'with DB'
+        describe options[:adapter] do # rubocop:disable RSpec/EmptyExampleGroup
+          self.instance_eval(&block)
+        end
+      end
+    end
+  end
+
+  # Set up the given database.
+  def setup_db(options)
+    ActiveRecord::Base.establish_connection(options)
+    migration_class_name = 'DbBackendMigration'
+    migration_version = '[5.2]'
+    migration = ERB.new(
+      File.read('lib/generators/deimos/db_backend/templates/migration')
+    ).result(binding)
+    eval(migration) # rubocop:disable Security/Eval
+    ActiveRecord::Migration.new.run(DbBackendMigration, direction: :up)
+
+    ActiveRecord::Base.descendants.each do |klass|
+      klass.reset_sequence_name if klass.respond_to?(:reset_sequence_name)
+      # reset internal variables - terrible hack to trick Rails into doing this
+      table_name = klass.table_name
+      klass.table_name = "#{table_name}2"
+      klass.table_name = table_name
+    end
+  end
+end
+
+RSpec.configure do |config|
+  config.extend(DbConfigs)
+  include DbConfigs
+  config.include TestRunners
+  config.full_backtrace = true
+
+  # true by default for RSpec 4.0
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+
+  config.before(:all) do
+    Time.zone = 'EST'
+    ActiveRecord::Base.logger = Logger.new('/dev/null')
+    ActiveRecord::Base.establish_connection(
+      'adapter' => 'sqlite3',
+      'database' => 'test.sqlite3'
+    )
+  end
+  config.include Deimos::TestHelpers
+  config.include ActiveSupport::Testing::TimeHelpers
+  config.before(:suite) do
+    Time.zone = 'EST'
+    ActiveRecord::Base.logger = Logger.new('/dev/null')
+    setup_db(DbConfigs::DB_OPTIONS.last)
+    Deimos.configure do |fr_config|
+      fr_config.phobos_config_file = File.join(File.dirname(__FILE__), 'phobos.yml')
+      fr_config.schema_path = File.join(File.expand_path(__dir__), 'schemas')
+      fr_config.reraise_consumer_errors = true
+      fr_config.schema_registry_url = ENV['SCHEMA_REGISTRY'] || 'http://localhost:8081'
+      fr_config.seed_broker = ENV['KAFKA_SEED_BROKER'] || 'localhost:9092'
+      fr_config.logger = Logger.new('/dev/null')
+
+      # Use Mock Metrics and Tracing for rspecs
+      fr_config.metrics = Deimos::Metrics::Mock.new
+      fr_config.tracer = Deimos::Tracing::Mock.new
+    end
+  end
+
+  config.before(:each) do |ex|
+    stub_producers_and_consumers! unless ex.metadata[:integration]
+
+    @previous_config = Deimos.config.dup
+    @previous_phobos_config = Phobos.config.dup
+  end
+
+  config.after(:each) do
+    Deimos.config = @previous_config
+    Phobos.instance_variable_set(:@config, @previous_phobos_config)
+  end
+
+end
+
+RSpec.shared_context('with DB') do
+  before(:all) do
+    setup_db(self.class.metadata[:db_config] || DbConfigs::DB_OPTIONS.last)
+  end
+
+  after(:each) do
+    Deimos::KafkaMessage.delete_all
+    Deimos::KafkaTopicInfo.delete_all
+  end
+end
+
+RSpec.shared_context('with publish_backend') do
+  before(:each) do
+    producer_class = Class.new(Deimos::Producer) do
+      schema 'MySchema'
+      namespace 'com.my-namespace'
+      topic 'my-topic'
+      key_config field: 'test_id'
+    end
+    stub_const('MyProducer', producer_class)
+
+    producer_class = Class.new(Deimos::Producer) do
+      schema 'MySchema'
+      namespace 'com.my-namespace'
+      topic 'my-topic'
+      key_config none: true
+    end
+    stub_const('MyNoKeyProducer', producer_class)
+  end
+
+  let(:messages) do
+    (1..3).map do |i|
+      build_message({ foo: i }, 'my-topic', "foo#{i}")
+    end
+  end
+end

data/spec/updateable_schema_store_spec.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+describe AvroTurf::SchemaStore do
+
+  it 'should add an in-memory schema' do
+    schema_store = described_class.new(path: Deimos.config.schema_path)
+    schema_store.load_schemas!
+    found_schema = schema_store.find('MySchema', 'com.my-namespace').as_json
+    expect(found_schema['name']).to eq('MySchema')
+    expect(found_schema['namespace']).to eq('com.my-namespace')
+    expect(found_schema['fields'].size).to eq(2)
+    expect(found_schema['fields'][0]['type']['type_sym']).to eq('string')
+    expect(found_schema['fields'][0]['name']).to eq('test_id')
+    new_schema = {
+      'namespace' => 'com.my-namespace',
+      'name' => 'MyNewSchema',
+      'type' => 'record',
+      'doc' => 'Test schema',
+      'fields' => [
+        {
+          'name' => 'my_id',
+          'type' => 'int',
+          'doc' => 'test int'
+        }
+      ]
+    }
+    schema_store.add_schema(new_schema)
+    found_schema = schema_store.find('MyNewSchema', 'com.my-namespace').
+      as_json
+    expect(found_schema['name']).to eq('MyNewSchema')
+    expect(found_schema['namespace']).to eq('com.my-namespace')
+    expect(found_schema['fields'].size).to eq(1)
+    expect(found_schema['fields'][0]['type']['type_sym']).to eq('int')
+    expect(found_schema['fields'][0]['name']).to eq('my_id')
+  end
+end