flipp-ruby-kafka-temp-fork 0.0.1

data/spec/integration/send_kafka_spec.rb

@@ -0,0 +1,63 @@
+require 'net/http'
+
+# frozen_string_literal: true
+
+RSpec.describe 'integration spec', integration: true do # rubocop:disable RSpec/DescribeClass
+  let(:producer_class) do
+    Class.new(FlippRubyKafka::Producer) do
+      schema 'MySchema'
+      namespace 'com.my-namespace'
+      topic 'my_consume_topic'
+      key_config field: 'test_id'
+    end
+  end
+
+  let(:producer_class_with_id) do
+    Class.new(FlippRubyKafka::Producer) do
+      schema 'MySchemaWithId'
+      namespace 'com.my-namespace'
+      topic 'my_consume_topic_with_id'
+      key_config schema: 'MySchema-key'
+    end
+  end
+
+  prepend_before(:each) do
+    kafkateria_delete_topic('my_consume_topic')
+  end
+
+  it 'should publish a keyed Kafka msg' do
+    payload = { 'test_id' => 'foo', 'some_int' => 5 }
+    producer_class.publish(payload)
+    sleep(1)
+    messages = kafkateria_get_messages(topic: 'my_consume_topic')
+    puts messages
+    expect(messages.size).to eq(1)
+    expect(messages[0][:payload]).to eq(payload)
+  end
+
+  it 'should send original and updated kafka messages for non keyed Kafka msg' do
+    payload = { 'test_id' => 'foobar', 'some_int' => 6 }
+    payload_update = { 'test_id' => 'foobar', 'some_int' => 7 }
+    producer_class.publish(payload)
+    producer_class.publish(payload_update)
+    sleep(1)
+    messages = kafkateria_get_messages(topic: 'my_consume_topic')
+    puts messages.size
+    expect(messages.size).to eq(2)
+    expect(messages[0][:payload]).to eq(payload)
+    expect(messages[1][:payload]).to eq(payload_update)
+  end
+
+  it 'should send only updated kafka messages for keyed Kafka msg' do
+    payload = { 'test_id' => 'foo', 'some_int' => 6, :payload_key => { 'test_id' => 'foo_key' } }
+    payload_update = { 'test_id' => 'foobar', 'some_int' => 7, :payload_key => { 'test_id' => 'foo_key' } }
+    producer_class_with_id.publish(payload)
+    producer_class_with_id.publish(payload_update)
+    sleep(1)
+    messages = kafkateria_get_messages(topic: 'my_consume_topic_with_id')
+    puts messages.size
+    expect(messages.size).to eq(1)
+    expect(messages[0][:payload]).to eq(payload_update)
+  end
+
+end

data/spec/phobos.yml

@@ -0,0 +1,73 @@
+logger:
+  # Optional log file, set to false or remove to disable it
+  file: log/phobos.log
+  # Optional output format for stdout, default is false (human readable).
+  # Set to true to enable json output.
+  stdout_json: false
+  level: debug
+  # Comment the block to disable ruby-kafka logs
+  ruby_kafka:
+    level: debug
+
+kafka:
+  # identifier for this application
+  client_id: phobos
+  # timeout setting for connecting to brokers
+  connect_timeout: 15
+  # timeout setting for socket connections
+  socket_timeout: 15
+
+producer:
+  # number of seconds a broker can wait for replicas to acknowledge
+  # a write before responding with a timeout
+  ack_timeout: 5
+  # number of replicas that must acknowledge a write, or `:all`
+  # if all in-sync replicas must acknowledge
+  required_acks: :all
+  # number of retries that should be attempted before giving up sending
+  # messages to the cluster. Does not include the original attempt
+  max_retries: 2
+  # number of seconds to wait between retries
+  retry_backoff: 1
+  # number of messages allowed in the buffer before new writes will
+  # raise {BufferOverflow} exceptions
+  max_buffer_size: 10000
+  # maximum size of the buffer in bytes. Attempting to produce messages
+  # when the buffer reaches this size will result in {BufferOverflow} being raised
+  max_buffer_bytesize: 10000000
+  # name of the compression codec to use, or nil if no compression should be performed.
+  # Valid codecs: `:snappy` and `:gzip`
+  compression_codec:
+  # number of messages that needs to be in a message set before it should be compressed.
+  # Note that message sets are per-partition rather than per-topic or per-producer
+  compression_threshold: 1
+  # maximum number of messages allowed in the queue. Only used for async_producer
+  max_queue_size: 10000
+  # if greater than zero, the number of buffered messages that will automatically
+  # trigger a delivery. Only used for async_producer
+  delivery_threshold: 0
+  # if greater than zero, the number of seconds between automatic message
+  # deliveries. Only used for async_producer
+  delivery_interval: 0
+
+consumer:
+  # number of seconds after which, if a client hasn't contacted the Kafka cluster,
+  # it will be kicked out of the group
+  session_timeout: 300
+  # interval between offset commits, in seconds
+  offset_commit_interval: 10
+  # number of messages that can be processed before their offsets are committed.
+  # If zero, offset commits are not triggered by message processing
+  offset_commit_threshold: 0
+  # interval between heartbeats; must be less than the session window
+  heartbeat_interval: 10
+
+backoff:
+  min_ms: 1000
+  max_ms: 60000
+
+listeners:
+  - handler: ConsumerTest::MyConsumer
+    topic: my_consume_topic
+    group_id: my_group_id
+    max_bytes_per_partition: 524288 # 512 KB

data/spec/schemas/com/my-namespace/MySchema-key.avsc

@@ -0,0 +1,13 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MySchema-key",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "test_id",
+      "type": "string",
+      "doc": "test string"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/MySchema.avsc

@@ -0,0 +1,18 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MySchema",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "test_id",
+      "type": "string",
+      "doc": "test string"
+    },
+    {
+      "name": "some_int",
+      "type": "int",
+      "doc": "test int"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc

@@ -0,0 +1,33 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MySchemaWithDateTimes",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "test_id",
+      "type": "string",
+      "doc": "test string"
+    },
+    {
+      "name": "updated_at",
+      "type": ["int", "null"]
+    },
+    {
+      "name": "some_int",
+      "type": ["null", "int"],
+      "doc": "test int"
+    },
+    {
+      "name": "some_datetime_int",
+      "type": ["null", "int"],
+      "doc": "test datetime"
+    },
+    {
+      "name": "timestamp",
+      "type": "string",
+      "doc": "ISO timestamp",
+      "default": ""
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/MySchemaWithId.avsc

@@ -0,0 +1,28 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MySchemaWithId",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "test_id",
+      "type": "string",
+      "doc": "test string"
+    },
+    {
+      "name": "some_int",
+      "type": "int",
+      "doc": "test int"
+    },
+    {
+      "name": "message_id",
+      "type": "string",
+      "doc": "UUID"
+    },
+    {
+      "name": "timestamp",
+      "type": "string",
+      "doc": "timestamp"
+    }
+  ]
+}

data/spec/spec_helper.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
+require 'flipp_ruby_kafka'
+require 'flipp_ruby_kafka/test_helpers'
+require 'deimos/test_helpers'
+require 'deimos/metrics/mock'
+require 'deimos/tracing/mock'
+require 'active_support/testing/time_helpers'
+
+RSpec.configure do |config|
+  config.full_backtrace = true
+
+  # true by default for RSpec 4.0
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+
+  config.include FlippRubyKafka::TestHelpers
+  config.include ActiveSupport::Testing::TimeHelpers
+  config.before(:suite) do
+    Time.zone = 'EST'
+  end
+
+  config.before(:each) do |ex|
+    stub_producers_and_consumers! unless ex.metadata[:integration]
+
+    FlippRubyKafka.configure do
+      logger ActiveSupport::TaggedLogging.new(Logger.new(STDOUT))
+      metrics Deimos::Metrics::Mock
+      tracer Deimos::Tracing::Mock
+      schema.path(File.join(File.expand_path(__dir__), 'schemas'))
+      kafkateria_url ENV['KAFKATERIA_URL'] || 'http://localhost:3000'
+    end
+  end
+
+  config.after(:each) do
+    FlippRubyKafka.config.reset!
+  end
+
+end

data/spec/utils/platform_schema_validation_spec.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'flipp_ruby_kafka/utils/platform_schema_validation'
+
+describe(FlippRubyKafka::Utils::PlatformSchemaValidation) do
+  prepend_before(:each) do
+    # :nodoc:
+    consumer_class = Class.new(FlippRubyKafka::Consumer) do
+      # :nodoc:
+      def consume
+      end
+    end
+    stub_const('ConsumerTest::MyConsumer', consumer_class)
+
+    producer_class = Class.new(FlippRubyKafka::Producer)
+    stub_const('MyProducer', producer_class)
+  end
+
+  before(:each) do
+
+    FlippRubyKafka.configure do
+      producer do
+        class_name 'MyProducer'
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        topic 'my-topic'
+        key_config field: 'test_id'
+      end
+      consumer do
+        class_name 'ConsumerTest::MyConsumer'
+        schema 'MySchemaWithDateTimes'
+        namespace 'com.my-namespace'
+        key_config plain: true
+        topic 'my_consume_topic'
+        group_id 'my-group-id'
+      end
+    end
+
+  end
+
+  specify '#generate_config' do
+    base_path = File.join(File.expand_path('..', __dir__), 'schemas')
+    validator = described_class.new(base_path: base_path)
+    config = validator.generate_config
+    expect(config[:environments][:default]).to include(
+      'file' => "#{base_path}/com/my-namespace/MySchema.avsc",
+      'subject' => 'my-topic-value'
+    )
+    expect(config[:environments][:default]).to include(
+      'file' => "#{base_path}/com/my-namespace/MySchemaWithDateTimes.avsc",
+      'subject' => 'my_consume_topic-value'
+    )
+  end
+end

data/spec/utils/platform_topic_validation_spec.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'flipp_ruby_kafka/utils/platform_topic_validation'
+
+describe(FlippRubyKafka::Utils::PlatformTopicValidation) do
+  prepend_before(:each) do
+    # :nodoc:
+    consumer_class = Class.new(FlippRubyKafka::Consumer) do
+      # :nodoc:
+      def consume
+      end
+    end
+    stub_const('ConsumerTest::MyConsumer', consumer_class)
+
+    producer_class = Class.new(FlippRubyKafka::Producer)
+    stub_const('MyProducer', producer_class)
+  end
+
+  before(:each) do
+
+    FlippRubyKafka.configure do
+      producer do
+        class_name 'MyProducer'
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        topic 'my-topic'
+        key_config field: 'test_id'
+      end
+      consumer do
+        class_name 'ConsumerTest::MyConsumer'
+        schema 'MySchemaWithDateTimes'
+        namespace 'com.my-namespace'
+        key_config plain: true
+        topic 'my_consume_topic'
+        group_id 'my-group-id'
+      end
+    end
+
+  end
+
+  specify '#generate_config' do
+    config = described_class.generate_config('my-service')
+    expect(config).
+      to eq(
+        acls: [{ role: 'producer', topic: 'my-topic' },
+               { role: 'consumer', topic: 'my_consume_topic' }],
+        service: 'my-service'
+      )
+  end
+end

metadata

@@ -0,0 +1,262 @@
+--- !ruby/object:Gem::Specification
+name: flipp-ruby-kafka-temp-fork
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Daniel Orner
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-08-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ddtrace
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.11'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.11'
+- !ruby/object:Gem::Dependency
+  name: deimos-temp-fork
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+- !ruby/object:Gem::Dependency
+  name: flipp-ruby-dogstatsd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: flipp_ruby_style
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+- !ruby/object:Gem::Dependency
+  name: guard-rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: rspec_junit_formatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.58'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.58'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.27'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.27'
+description: Write a longer description. Optional.
+email:
+- daniel.orner@wishabi.com
+executables:
+- flipp_ruby_kafka
+extensions: []
+extra_rdoc_files: []
+files:
+- ".circleci/config.yml"
+- ".dockerignore"
+- ".gitignore"
+- ".gitmodules"
+- ".rspec"
+- ".rubocop.yml"
+- ".ruby-gemset"
+- ".ruby-version"
+- CHANGELOG.md
+- Dockerfile
+- Gemfile
+- Gemfile.lock
+- Guardfile
+- README.md
+- Rakefile
+- bin/flipp_ruby_kafka
+- deploy/artifactory.sh
+- deploy/config.json
+- docker-compose.yml
+- docs/UPGRADE_1.0.md
+- flipp-ruby-kafka.gemspec
+- kafkateria/factories/my_schema.rb
+- kafkateria/factories/my_schema_with_id.rb
+- lib/flipp_ruby_kafka.rb
+- lib/flipp_ruby_kafka/flipp_datadog_metrics.rb
+- lib/flipp_ruby_kafka/test_helpers.rb
+- lib/flipp_ruby_kafka/utils/platform_schema_validation.rb
+- lib/flipp_ruby_kafka/utils/platform_topic_validation.rb
+- lib/flipp_ruby_kafka/version.rb
+- lib/generators/flipp_ruby_kafka/schema_validation/schema_validation_generator.rb
+- lib/generators/flipp_ruby_kafka/topic_validation/topic_validation_generator.rb
+- spec/flipp_ruby_kafka_spec.rb
+- spec/integration/send_kafka_spec.rb
+- spec/phobos.yml
+- spec/schemas/com/my-namespace/MySchema-key.avsc
+- spec/schemas/com/my-namespace/MySchema.avsc
+- spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
+- spec/schemas/com/my-namespace/MySchemaWithId.avsc
+- spec/spec_helper.rb
+- spec/utils/platform_schema_validation_spec.rb
+- spec/utils/platform_topic_validation_spec.rb
+homepage: ''
+licenses: []
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.3
+signing_key:
+specification_version: 4
+summary: Flipp Kafka libraries for Ruby.
+test_files:
+- spec/flipp_ruby_kafka_spec.rb
+- spec/integration/send_kafka_spec.rb
+- spec/phobos.yml
+- spec/schemas/com/my-namespace/MySchema-key.avsc
+- spec/schemas/com/my-namespace/MySchema.avsc
+- spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
+- spec/schemas/com/my-namespace/MySchemaWithId.avsc
+- spec/spec_helper.rb
+- spec/utils/platform_schema_validation_spec.rb
+- spec/utils/platform_topic_validation_spec.rb