logstash-integration-kafka 10.6.0-java → 10.7.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d8b40d779e91e9c05dece65249660ab5c272b6833658cbca51b977d92936f42
-  data.tar.gz: 58323e216be645aede9f0b49c27824958b6627485bdf79a6774cd5f87b818245
+  metadata.gz: 166b8a4d60aaa51f4c22e6ba98f5ecf9c5991ce853c3dfa8ae15c85ad28ad4bd
+  data.tar.gz: 1fb417190daa76b08596d8c35df7004b0f1ea284eff347ab7bd5eaeabd004e0d
 SHA512:
-  metadata.gz: 0f05eec028758745a2ab04b90d721128c088c46d0bb9a01923c389118c99a718a561d2ca2420fc2e206e4cf75e4e3695e1704a17d9eff9f4f18e66da3d3ccb85
-  data.tar.gz: 85e117a64d14d013674869ccadfb5487a04991ec83c6a9d6874496b862be200bf4e7203d3ac4fc779d67131ecbe82212fafd66884114337d17ed33dda7ad0963
+  metadata.gz: 25150a143b4ccd2375572697df7bafadc87b35930f11e9916a7db1a25a69830ab1e3ae4a2d32119bc127c81af557af277188ef705fed24d1c8f1929e5bc1e73d
+  data.tar.gz: 845de0baed1db54a6ec2901b92665deca90a21f40b00400a5ae5490d6355f3771048547b158dbe19b8c7ac66587398640052c6295546793ee95251df276e35af

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 10.7.0
+  - Switched use from Faraday to Manticore as HTTP client library to access Schema Registry service 
+    to fix issue [#63](https://github.com/logstash-plugins/logstash-integration-kafka/issue/63) 
+
 ## 10.6.0
   - Added functionality to Kafka input to use Avro deserializer in retrieving data from Kafka. The schema is retrieved
     from an instance of Confluent's Schema Registry service [#51](https://github.com/logstash-plugins/logstash-integration-kafka/pull/51)

data/lib/logstash/inputs/kafka.rb

@@ -4,7 +4,7 @@ require 'stud/interval'
 require 'java'
 require 'logstash-integration-kafka_jars.rb'
 require 'logstash/plugin_mixins/kafka_support'
-require "faraday"
+require 'manticore'
 require "json"
 require "logstash/json"
 require_relative '../plugin_mixins/common'

data/lib/logstash/plugin_mixins/common.rb

@@ -45,20 +45,21 @@ module LogStash
 
       private
       def check_for_schema_registry_connectivity_and_subjects
-        client = Faraday.new(@schema_registry_url.to_s) do |conn|
-          if schema_registry_proxy && !schema_registry_proxy.empty?
-            conn.proxy = schema_registry_proxy.to_s
-          end
-          if schema_registry_key and !schema_registry_key.empty?
-            conn.basic_auth(schema_registry_key, schema_registry_secret.value)
-          end
+        options = {}
+        if schema_registry_proxy && !schema_registry_proxy.empty?
+          options[:proxy] = schema_registry_proxy.to_s
         end
+        if schema_registry_key and !schema_registry_key.empty?
+          options[:auth] = {:user => schema_registry_key, :password => schema_registry_secret.value}
+        end
+        client = Manticore::Client.new(options)
+
         begin
-          response = client.get('/subjects')
-        rescue Faraday::Error => e
+          response = client.get(@schema_registry_url.to_s + '/subjects').body
+        rescue Manticore::ManticoreException => e
           raise LogStash::ConfigurationError.new("Schema registry service doesn't respond, error: #{e.message}")
         end
-        registered_subjects = JSON.parse response.body
+        registered_subjects = JSON.parse response
         expected_subjects = @topics.map { |t| "#{t}-value"}
         if (expected_subjects & registered_subjects).size != expected_subjects.size
           undefined_topic_subjects = expected_subjects - registered_subjects

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.6.0'
+  s.version         = '10.7.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -46,6 +46,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-codec-json'
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1.0'
+  s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'rspec-wait'

data/spec/integration/inputs/kafka_spec.rb

@@ -3,7 +3,7 @@ require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/kafka"
 require "rspec/wait"
 require "stud/try"
-require "faraday"
+require "manticore"
 require "json"
 
 # Please run kafka_test_setup.sh prior to executing this integration test.
@@ -164,11 +164,11 @@ describe "schema registry connection options" do
 
       before(:each) do
         response = save_avro_schema_to_schema_registry(File.join(Dir.pwd, "spec", "unit", "inputs", "avro_schema_fixture_payment.asvc"), SUBJECT_NAME)
-        expect( response.status ).to be(200)
+        expect( response.code ).to be(200)
       end
 
       after(:each) do
-        schema_registry_client = Faraday.new('http://localhost:8081')
+        schema_registry_client = Manticore::Client.new
         delete_remote_schema(schema_registry_client, SUBJECT_NAME)
       end
 
@@ -187,26 +187,26 @@ end
 def save_avro_schema_to_schema_registry(schema_file, subject_name)
   raw_schema = File.readlines(schema_file).map(&:chomp).join
   raw_schema_quoted = raw_schema.gsub('"', '\"')
-  response = Faraday.post("http://localhost:8081/subjects/#{subject_name}/versions",
-          '{"schema": "' + raw_schema_quoted + '"}',
-          "Content-Type" => "application/vnd.schemaregistry.v1+json")
+  response = Manticore.post("http://localhost:8081/subjects/#{subject_name}/versions",
+          body: '{"schema": "' + raw_schema_quoted + '"}',
+          headers: {"Content-Type" => "application/vnd.schemaregistry.v1+json"})
   response
 end
 
 def delete_remote_schema(schema_registry_client, subject_name)
-  expect(schema_registry_client.delete("/subjects/#{subject_name}").status ).to be(200)
-  expect(schema_registry_client.delete("/subjects/#{subject_name}?permanent=true").status ).to be(200)
+  expect(schema_registry_client.delete("http://localhost:8081/subjects/#{subject_name}").code ).to be(200)
+  expect(schema_registry_client.delete("http://localhost:8081/subjects/#{subject_name}?permanent=true").code ).to be(200)
 end
 
 # AdminClientConfig = org.alpache.kafka.clients.admin.AdminClientConfig
 
 describe "Schema registry API", :integration => true do
 
-  let(:schema_registry) { Faraday.new('http://localhost:8081') }
+  let(:schema_registry) { Manticore::Client.new }
 
   context 'listing subject on clean instance' do
     it "should return an empty set" do
-      subjects = JSON.parse schema_registry.get('/subjects').body
+      subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
       expect( subjects ).to be_empty
     end
   end
@@ -214,30 +214,30 @@ describe "Schema registry API", :integration => true do
   context 'send a schema definition' do
     it "save the definition" do
       response = save_avro_schema_to_schema_registry(File.join(Dir.pwd, "spec", "unit", "inputs", "avro_schema_fixture_payment.asvc"), "schema_test_1")
-      expect( response.status ).to be(200)
+      expect( response.code ).to be(200)
       delete_remote_schema(schema_registry, "schema_test_1")
     end
 
     it "delete the schema just added" do
       response = save_avro_schema_to_schema_registry(File.join(Dir.pwd, "spec", "unit", "inputs", "avro_schema_fixture_payment.asvc"), "schema_test_1")
-      expect( response.status ).to be(200)
+      expect( response.code ).to be(200)
 
-      expect( schema_registry.delete('/subjects/schema_test_1?permanent=false').status ).to be(200)
+      expect( schema_registry.delete('http://localhost:8081/subjects/schema_test_1?permanent=false').code ).to be(200)
       sleep(1)
-      subjects = JSON.parse schema_registry.get('/subjects').body
+      subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
       expect( subjects ).to be_empty
     end
   end
 
   context 'use the schema to serialize' do
     after(:each) do
-      expect( schema_registry.delete('/subjects/topic_avro-value').status ).to be(200)
+      expect( schema_registry.delete('http://localhost:8081/subjects/topic_avro-value').code ).to be(200)
       sleep 1
-      expect( schema_registry.delete('/subjects/topic_avro-value?permanent=true').status ).to be(200)
+      expect( schema_registry.delete('http://localhost:8081/subjects/topic_avro-value?permanent=true').code ).to be(200)
 
       Stud.try(3.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do
         wait(10).for do
-          subjects = JSON.parse schema_registry.get('/subjects').body
+          subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
           subjects.empty?
         end.to be_truthy
       end
@@ -299,7 +299,7 @@ describe "Schema registry API", :integration => true do
       delete_topic_if_exists avro_topic_name
       write_some_data_to avro_topic_name
 
-      subjects = JSON.parse schema_registry.get('/subjects').body
+      subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
       expect( subjects ).to contain_exactly("topic_avro-value")
 
       num_events = 1

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.6.0
+  version: 10.7.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-28 00:00:00.000000000 Z
+date: 2020-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -106,6 +106,26 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 0.1.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  name: manticore
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: