logstash-input-dis 1.1.3 → 1.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b1405b2ee6e1dd978e2954b3eb97b129fb96ade0
-  data.tar.gz: 76006db276e07f84a0a76163a99877382e7ba26c
+  metadata.gz: d6f2a35b566544b0d1f13ba1d3125947b46a42ce
+  data.tar.gz: d58456cbeaf297ec22392c03a1676e014251da15
 SHA512:
-  metadata.gz: 81739e648876455c17b35fdb4e7cd2c232730c051b9a554b96bd8ebcdafbbead1146f5d4ddc7c4b511a60f9dd84a9d84d3db3e374295358ac1eb53797d0222fd
-  data.tar.gz: 6bd2efaf31d1a050da999844a97bffaef2505e0e39934880dd04a78e91e23df82ff93a0424a6204bcca6f33acb8695fd6a1ed6e1714849df4aac431d7dd42288
+  metadata.gz: ee11eb3a2221424c139bca80972d236bb12011596745eb6efe2dddc1197caeda89f2ec3b425a1a358cd17732a2b9036d09dcfe6166d7f4369d3a6ed8e4fe3c60
+  data.tar.gz: 71c21e1ed24d203db4e2b189cf4d8e57a1a7851060dd43f32976701fd6760fd6de9fbce4b4c7bfe963389c54fa37ec3532b0a1ec228992f14a3ff12dc9639d97

data/README.md

@@ -7,7 +7,7 @@ This is a plugin for [Logstash](https://github.com/elastic/logstash).
 To get started using this plugin, you will need three things:
 
 1. JDK 1.8 +
-2. JRuby with the Bundler gem installed, 9.0.0.0 +
+2. JRuby with the Bundler gem installed, 9.0.0.0 ~ 9.2.0.0
 3. Maven
 4. Logstash
 

data/lib/logstash-input-dis_jars.rb

@@ -1,5 +1,54 @@
-# encoding: utf-8
-require 'logstash/environment'
+# this is a generated file, to avoid over-writing it just delete this comment
+begin
+  require 'jar_dependencies'
+rescue LoadError
+  require 'com/huaweicloud/dis/huaweicloud-dis-kafka-adapter-common/1.2.9/huaweicloud-dis-kafka-adapter-common-1.2.9.jar'
+  require 'com/huaweicloud/dis/huaweicloud-sdk-java-dis-iface/1.3.6/huaweicloud-sdk-java-dis-iface-1.3.6.jar'
+  require 'org/apache/httpcomponents/httpasyncclient/4.1.3/httpasyncclient-4.1.3.jar'
+  require 'org/apache/logging/log4j/log4j-api/2.8.2/log4j-api-2.8.2.jar'
+  require 'commons-io/commons-io/2.4/commons-io-2.4.jar'
+  require 'com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar'
+  require 'org/slf4j/slf4j-api/1.7.24/slf4j-api-1.7.24.jar'
+  require 'org/apache/logging/log4j/log4j-slf4j-impl/2.8.2/log4j-slf4j-impl-2.8.2.jar'
+  require 'com/huaweicloud/dis/huaweicloud-dis-kafka-adapter/1.2.9/huaweicloud-dis-kafka-adapter-1.2.9.jar'
+  require 'com/github/luben/zstd-jni/1.4.0-1/zstd-jni-1.4.0-1.jar'
+  require 'commons-logging/commons-logging/1.2/commons-logging-1.2.jar'
+  require 'com/fasterxml/jackson/core/jackson-annotations/2.9.8/jackson-annotations-2.9.8.jar'
+  require 'joda-time/joda-time/2.8.1/joda-time-2.8.1.jar'
+  require 'com/fasterxml/jackson/core/jackson-databind/2.9.8/jackson-databind-2.9.8.jar'
+  require 'org/apache/httpcomponents/httpmime/4.5.7/httpmime-4.5.7.jar'
+  require 'org/apache/httpcomponents/httpclient/4.5.7/httpclient-4.5.7.jar'
+  require 'commons-codec/commons-codec/1.11/commons-codec-1.11.jar'
+  require 'net/jpountz/lz4/lz4/1.3.0/lz4-1.3.0.jar'
+  require 'org/apache/httpcomponents/httpcore-nio/4.4.6/httpcore-nio-4.4.6.jar'
+  require 'com/fasterxml/jackson/core/jackson-core/2.9.8/jackson-core-2.9.8.jar'
+  require 'com/huaweicloud/dis/huaweicloud-sdk-java-dis/1.3.6/huaweicloud-sdk-java-dis-1.3.6.jar'
+  require 'org/apache/httpcomponents/httpcore/4.4.4/httpcore-4.4.4.jar'
+  require 'org/xerial/snappy/snappy-java/1.1.7.2/snappy-java-1.1.7.2.jar'
+end
 
-root_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
-LogStash::Environment.load_runtime_jars! File.join(root_dir, "vendor")
+if defined? Jars
+  require_jar 'com.huaweicloud.dis', 'huaweicloud-dis-kafka-adapter-common', '1.2.9'
+  require_jar 'com.huaweicloud.dis', 'huaweicloud-sdk-java-dis-iface', '1.3.6'
+  require_jar 'org.apache.httpcomponents', 'httpasyncclient', '4.1.3'
+  require_jar 'org.apache.logging.log4j', 'log4j-api', '2.8.2'
+  require_jar 'commons-io', 'commons-io', '2.4'
+  require_jar 'com.google.protobuf', 'protobuf-java', '2.5.0'
+  require_jar 'org.slf4j', 'slf4j-api', '1.7.24'
+  require_jar 'org.apache.logging.log4j', 'log4j-slf4j-impl', '2.8.2'
+  require_jar 'com.huaweicloud.dis', 'huaweicloud-dis-kafka-adapter', '1.2.9'
+  require_jar 'com.github.luben', 'zstd-jni', '1.4.0-1'
+  require_jar 'commons-logging', 'commons-logging', '1.2'
+  require_jar 'com.fasterxml.jackson.core', 'jackson-annotations', '2.9.8'
+  require_jar 'joda-time', 'joda-time', '2.8.1'
+  require_jar 'com.fasterxml.jackson.core', 'jackson-databind', '2.9.8'
+  require_jar 'org.apache.httpcomponents', 'httpmime', '4.5.7'
+  require_jar 'org.apache.httpcomponents', 'httpclient', '4.5.7'
+  require_jar 'commons-codec', 'commons-codec', '1.11'
+  require_jar 'net.jpountz.lz4', 'lz4', '1.3.0'
+  require_jar 'org.apache.httpcomponents', 'httpcore-nio', '4.4.6'
+  require_jar 'com.fasterxml.jackson.core', 'jackson-core', '2.9.8'
+  require_jar 'com.huaweicloud.dis', 'huaweicloud-sdk-java-dis', '1.3.6'
+  require_jar 'org.apache.httpcomponents', 'httpcore', '4.4.4'
+  require_jar 'org.xerial.snappy', 'snappy-java', '1.1.7.2'
+end

data/logstash-input-dis.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-dis'
-  s.version         = '1.1.3'
+  s.version         = '1.1.4'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads events from a DIS Stream"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'input'}
 
-  s.requirements << "jar 'com.huaweicloud.dis:huaweicloud-dis-kafka-adapter', '1.2.1'"
+  s.requirements << "jar 'com.huaweicloud.dis:huaweicloud-dis-kafka-adapter', '1.2.9'"
   s.requirements << "jar 'org.apache.logging.log4j:log4j-slf4j-impl', '2.8.2'"
 
   s.add_development_dependency 'jar-dependencies', '~> 0.3.2'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-dis
 version: !ruby/object:Gem::Version
-  version: 1.1.3
+  version: 1.1.4
 platform: ruby
 authors:
 - Data Ingestion Service
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-06-10 00:00:00.000000000 Z
+date: 2020-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -132,52 +132,55 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
-- lib/com/fasterxml/jackson/core/jackson-annotations/2.8.11/jackson-annotations-2.8.11.jar
-- lib/com/fasterxml/jackson/core/jackson-core/2.8.11/jackson-core-2.8.11.jar
-- lib/com/fasterxml/jackson/core/jackson-databind/2.8.11.3/jackson-databind-2.8.11.3.jar
+- lib/com/fasterxml/jackson/core/jackson-annotations/2.9.8/jackson-annotations-2.9.8.jar
+- lib/com/fasterxml/jackson/core/jackson-core/2.9.8/jackson-core-2.9.8.jar
+- lib/com/fasterxml/jackson/core/jackson-databind/2.9.8/jackson-databind-2.9.8.jar
+- lib/com/github/luben/zstd-jni/1.4.0-1/zstd-jni-1.4.0-1.jar
 - lib/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar
-- lib/com/huaweicloud/dis/huaweicloud-dis-kafka-adapter-common/1.2.1/huaweicloud-dis-kafka-adapter-common-1.2.1.jar
-- lib/com/huaweicloud/dis/huaweicloud-dis-kafka-adapter/1.2.1/huaweicloud-dis-kafka-adapter-1.2.1.jar
-- lib/com/huaweicloud/dis/huaweicloud-sdk-java-dis-iface/1.3.3/huaweicloud-sdk-java-dis-iface-1.3.3.jar
-- lib/com/huaweicloud/dis/huaweicloud-sdk-java-dis/1.3.3/huaweicloud-sdk-java-dis-1.3.3.jar
-- lib/commons-codec/commons-codec/1.9/commons-codec-1.9.jar
+- lib/com/huaweicloud/dis/huaweicloud-dis-kafka-adapter-common/1.2.9/huaweicloud-dis-kafka-adapter-common-1.2.9.jar
+- lib/com/huaweicloud/dis/huaweicloud-dis-kafka-adapter/1.2.9/huaweicloud-dis-kafka-adapter-1.2.9.jar
+- lib/com/huaweicloud/dis/huaweicloud-sdk-java-dis-iface/1.3.6/huaweicloud-sdk-java-dis-iface-1.3.6.jar
+- lib/com/huaweicloud/dis/huaweicloud-sdk-java-dis/1.3.6/huaweicloud-sdk-java-dis-1.3.6.jar
+- lib/commons-codec/commons-codec/1.11/commons-codec-1.11.jar
 - lib/commons-io/commons-io/2.4/commons-io-2.4.jar
 - lib/commons-logging/commons-logging/1.2/commons-logging-1.2.jar
 - lib/joda-time/joda-time/2.8.1/joda-time-2.8.1.jar
 - lib/logstash-input-dis_jars.rb
-- lib/logstash/inputs/dis.rb
+- lib/net/jpountz/lz4/lz4/1.3.0/lz4-1.3.0.jar
 - lib/org/apache/httpcomponents/httpasyncclient/4.1.3/httpasyncclient-4.1.3.jar
-- lib/org/apache/httpcomponents/httpclient/4.5.2/httpclient-4.5.2.jar
+- lib/org/apache/httpcomponents/httpclient/4.5.7/httpclient-4.5.7.jar
 - lib/org/apache/httpcomponents/httpcore-nio/4.4.6/httpcore-nio-4.4.6.jar
 - lib/org/apache/httpcomponents/httpcore/4.4.4/httpcore-4.4.4.jar
-- lib/org/apache/httpcomponents/httpmime/4.5.2/httpmime-4.5.2.jar
+- lib/org/apache/httpcomponents/httpmime/4.5.7/httpmime-4.5.7.jar
 - lib/org/apache/logging/log4j/log4j-api/2.8.2/log4j-api-2.8.2.jar
 - lib/org/apache/logging/log4j/log4j-slf4j-impl/2.8.2/log4j-slf4j-impl-2.8.2.jar
 - lib/org/slf4j/slf4j-api/1.7.24/slf4j-api-1.7.24.jar
 - lib/org/xerial/snappy/snappy-java/1.1.7.2/snappy-java-1.1.7.2.jar
 - logstash-input-dis.gemspec
 - spec/unit/inputs/dis_spec.rb
-- vendor/jar-dependencies/runtime-jars/commons-codec-1.9.jar
+- vendor/jar-dependencies/runtime-jars/commons-codec-1.11.jar
 - vendor/jar-dependencies/runtime-jars/commons-io-2.4.jar
 - vendor/jar-dependencies/runtime-jars/commons-logging-1.2.jar
 - vendor/jar-dependencies/runtime-jars/httpasyncclient-4.1.3.jar
-- vendor/jar-dependencies/runtime-jars/httpclient-4.5.2.jar
+- vendor/jar-dependencies/runtime-jars/httpclient-4.5.7.jar
 - vendor/jar-dependencies/runtime-jars/httpcore-4.4.4.jar
 - vendor/jar-dependencies/runtime-jars/httpcore-nio-4.4.6.jar
-- vendor/jar-dependencies/runtime-jars/httpmime-4.5.2.jar
-- vendor/jar-dependencies/runtime-jars/huaweicloud-dis-kafka-adapter-1.2.1.jar
-- vendor/jar-dependencies/runtime-jars/huaweicloud-dis-kafka-adapter-common-1.2.1.jar
-- vendor/jar-dependencies/runtime-jars/huaweicloud-sdk-java-dis-1.3.3.jar
-- vendor/jar-dependencies/runtime-jars/huaweicloud-sdk-java-dis-iface-1.3.3.jar
-- vendor/jar-dependencies/runtime-jars/jackson-annotations-2.8.11.jar
-- vendor/jar-dependencies/runtime-jars/jackson-core-2.8.11.jar
-- vendor/jar-dependencies/runtime-jars/jackson-databind-2.8.11.3.jar
+- vendor/jar-dependencies/runtime-jars/httpmime-4.5.7.jar
+- vendor/jar-dependencies/runtime-jars/huaweicloud-dis-kafka-adapter-1.2.9.jar
+- vendor/jar-dependencies/runtime-jars/huaweicloud-dis-kafka-adapter-common-1.2.9.jar
+- vendor/jar-dependencies/runtime-jars/huaweicloud-sdk-java-dis-1.3.6.jar
+- vendor/jar-dependencies/runtime-jars/huaweicloud-sdk-java-dis-iface-1.3.6.jar
+- vendor/jar-dependencies/runtime-jars/jackson-annotations-2.9.8.jar
+- vendor/jar-dependencies/runtime-jars/jackson-core-2.9.8.jar
+- vendor/jar-dependencies/runtime-jars/jackson-databind-2.9.8.jar
 - vendor/jar-dependencies/runtime-jars/joda-time-2.8.1.jar
 - vendor/jar-dependencies/runtime-jars/log4j-api-2.8.2.jar
 - vendor/jar-dependencies/runtime-jars/log4j-slf4j-impl-2.8.2.jar
+- vendor/jar-dependencies/runtime-jars/lz4-1.3.0.jar
 - vendor/jar-dependencies/runtime-jars/protobuf-java-2.5.0.jar
 - vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.24.jar
 - vendor/jar-dependencies/runtime-jars/snappy-java-1.1.7.2.jar
+- vendor/jar-dependencies/runtime-jars/zstd-jni-1.4.0-1.jar
 homepage: https://www.huaweicloud.com/product/dis.html
 licenses:
 - Apache License (2.0)
@@ -199,7 +202,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
-- jar 'com.huaweicloud.dis:huaweicloud-dis-kafka-adapter', '1.2.1'
+- jar 'com.huaweicloud.dis:huaweicloud-dis-kafka-adapter', '1.2.9'
 - jar 'org.apache.logging.log4j:log4j-slf4j-impl', '2.8.2'
 rubyforge_project:
 rubygems_version: 2.6.14.1

data/lib/logstash/inputs/dis.rb

@@ -1,205 +0,0 @@
-require 'logstash/namespace'
-require 'logstash/inputs/base'
-require 'stud/interval'
-require 'java'
-require 'logstash-input-dis_jars.rb'
-
-# This input will read events from a DIS stream, using DIS Kafka Adapter.
-class LogStash::Inputs::Dis < LogStash::Inputs::Base
-  config_name 'dis'
-
-  default :codec, 'plain'
-
-  config :default_trusted_jks_enabled, :validate => :boolean, :default => false
-  config :security_token, :validate => :string
-  config :exception_retries, :validate => :number, :default => 8
-  config :records_retries, :validate => :number, :default => 20
-  config :proxy_host, :validate => :string
-  config :proxy_port, :validate => :number, :default => 80
-  config :proxy_protocol, :validate => ["http", "https"], :default => "http"
-  config :proxy_username, :validate => :string
-  config :proxy_password, :validate => :string
-  config :proxy_workstation, :validate => :string
-  config :proxy_domain, :validate => :string
-  config :proxy_non_proxy_hosts, :validate => :string
-
-  # The frequency in milliseconds that the consumer offsets are committed to Kafka.
-  config :auto_commit_interval_ms, :validate => :string, :default => "5000"
-  # What to do when there is no initial offset in Kafka or if an offset is out of range:
-  #
-  # * earliest: automatically reset the offset to the earliest offset
-  # * latest: automatically reset the offset to the latest offset
-  # * none: throw exception to the consumer if no previous offset is found for the consumer's group
-  # * anything else: throw exception to the consumer.
-  config :auto_offset_reset, :validate => :string
-  # The id string to pass to the server when making requests. The purpose of this
-  # is to be able to track the source of requests beyond just ip/port by allowing
-  # a logical application name to be included.
-  config :client_id, :validate => :string, :default => "logstash"
-  # Ideally you should have as many threads as the number of partitions for a perfect
-  # balance — more threads than partitions means that some threads will be idle
-  config :consumer_threads, :validate => :number, :default => 1
-  # If true, periodically commit to Kafka the offsets of messages already returned by the consumer. 
-  # This committed offset will be used when the process fails as the position from
-  # which the consumption will begin.
-  config :enable_auto_commit, :validate => :string, :default => "true"
-  # The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber
-  # that happens to be made up of multiple processors. Messages in a topic will be distributed to all
-  # Logstash instances with the same `group_id`
-  config :group_id, :validate => :string, :default => "logstash"
-  # Java Class used to deserialize the record's key
-  config :key_deserializer_class, :validate => :string, :default => "com.huaweicloud.dis.adapter.kafka.common.serialization.StringDeserializer"
-  # Java Class used to deserialize the record's value
-  config :value_deserializer_class, :validate => :string, :default => "com.huaweicloud.dis.adapter.kafka.common.serialization.StringDeserializer"
-  # A list of streams to subscribe to, defaults to ["logstash"].
-  config :streams, :validate => :array, :default => ["logstash"]
-  # DIS Gateway endpoint
-  config :endpoint, :validate => :string, :default => "https://dis.cn-north-1.myhuaweicloud.com"
-  # The ProjectId of the specified region, it can be obtained from My Credential Page
-  config :project_id, :validate => :string
-  # Specifies use which region of DIS, now DIS only support cn-north-1
-  config :region, :validate => :string, :default => "cn-north-1"
-  # The Access Key ID for hwclouds, it can be obtained from My Credential Page
-  config :ak, :validate => :string, :required => true
-  # The Secret key ID is encrypted or not
-  config :is_sk_encrypted, :default => false
-  # The encrypt key used to encypt the Secret Key Id
-  config :encrypt_key, :validate => :string
-  # The Secret Key ID for hwclouds, it can be obtained from My Credential Page
-  config :sk, :validate => :string, :required => true
-  # A topic regex pattern to subscribe to. 
-  # The topics configuration will be ignored when using this configuration.
-  config :topics_pattern, :validate => :string
-  # Time kafka consumer will wait to receive new messages from topics
-  config :poll_timeout_ms, :validate => :number, :default => 100
-  # Option to add DIS metadata like stream, message size to the event.
-  # This will add a field named `dis` to the logstash event containing the following attributes:
-  #   `stream`: The stream this message is associated with
-  #   `consumer_group`: The consumer group used to read in this event
-  #   `partition`: The partition this message is associated with
-  #   `offset`: The offset from the partition this message is associated with
-  #   `key`: A ByteBuffer containing the message key
-  #   `timestamp`: The timestamp of this message
-  config :decorate_events, :validate => :boolean, :default => false
-
-
-  public
-  def register
-    @runner_threads = []
-  end # def register
-
-  public
-  def run(logstash_queue)
-    @runner_consumers = consumer_threads.times.map { |i| create_consumer("#{client_id}-#{i}") }
-    @runner_threads = @runner_consumers.map { |consumer| thread_runner(logstash_queue, consumer) }
-    @runner_threads.each { |t| t.join }
-  end # def run
-
-  public
-  def stop
-    @runner_consumers.each { |c| c.wakeup }
-  end
-
-  public
-  def kafka_consumers
-    @runner_consumers
-  end
-
-  private
-  def thread_runner(logstash_queue, consumer)
-    Thread.new do
-      begin
-        unless @topics_pattern.nil?
-          nooplistener = com.huaweicloud.dis.adapter.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener.new
-          pattern = java.util.regex.Pattern.compile(@topics_pattern)
-          consumer.subscribe(pattern, nooplistener)
-        else
-          consumer.subscribe(streams);
-        end
-        codec_instance = @codec.clone
-        while !stop?
-          records = consumer.poll(poll_timeout_ms)
-          for record in records do
-            codec_instance.decode(record.value.to_s) do |event|
-              decorate(event)
-              if @decorate_events
-                event.set("[@metadata][dis][topic]", record.topic)
-                event.set("[@metadata][dis][consumer_group]", @group_id)
-                event.set("[@metadata][dis][partition]", record.partition)
-                event.set("[@metadata][dis][offset]", record.offset)
-                event.set("[@metadata][dis][key]", record.key)
-                event.set("[@metadata][dis][timestamp]", record.timestamp)
-              end
-              logstash_queue << event
-            end
-          end
-          # Manual offset commit
-          if @enable_auto_commit == "false"
-            consumer.commitSync
-          end
-        end
-      rescue org.apache.kafka.common.errors.WakeupException => e
-        raise e if !stop?
-      ensure
-        consumer.close
-      end
-    end
-  end
-
-  private
-  def create_consumer(client_id)
-    begin
-      props = java.util.Properties.new
-      kafka = com.huaweicloud.dis.adapter.kafka.clients.consumer.ConsumerConfig
-
-      props.put("IS_DEFAULT_TRUSTED_JKS_ENABLED", default_trusted_jks_enabled.to_s)
-      props.put("security.token", security_token) unless security_token.nil?
-      props.put("exception.retries", exception_retries.to_s)
-      props.put("records.retries", records_retries.to_s)
-      props.put("PROXY_HOST", proxy_host) unless proxy_host.nil?
-      props.put("PROXY_PORT", proxy_port.to_s)
-      props.put("PROXY_PROTOCOL", proxy_protocol)
-      props.put("PROXY_USERNAME", proxy_username) unless proxy_username.nil?
-      props.put("PROXY_PASSWORD", proxy_password) unless proxy_password.nil?
-      props.put("PROXY_WORKSTATION", proxy_workstation) unless proxy_workstation.nil?
-      props.put("PROXY_DOMAIN", proxy_domain) unless proxy_domain.nil?
-      props.put("NON_PROXY_HOSTS", proxy_non_proxy_hosts) unless proxy_non_proxy_hosts.nil?
-      
-      props.put("auto.commit.interval.ms", auto_commit_interval_ms)
-      props.put("auto.offset.reset", auto_offset_reset) unless auto_offset_reset.nil?
-      props.put("client.id", client_id)
-      props.put("enable.auto.commit", enable_auto_commit)
-      props.put("group.id", group_id)
-      props.put("key.deserializer", "com.huaweicloud.dis.adapter.kafka.common.serialization.StringDeserializer")
-      props.put("value.deserializer", "com.huaweicloud.dis.adapter.kafka.common.serialization.StringDeserializer")
-
-      # endpoint, project_id, region, ak, sk
-      props.put("endpoint", endpoint)
-      props.put("projectId", project_id)
-      props.put("region", region)
-      props.put("ak", ak)
-      if is_sk_encrypted
-        decrypted_sk = decrypt(@sk)
-        props.put("sk", decrypted_sk)
-      else
-        props.put("sk", sk)
-      end
-
-      com.huaweicloud.dis.adapter.kafka.clients.consumer.DISKafkaConsumer.new(props)
-    rescue => e
-      logger.error("Unable to create DIS Kafka consumer from given configuration",
-                   :kafka_error_message => e,
-                   :cause => e.respond_to?(:getCause) ? e.getCause() : nil)
-      throw e
-    end
-  end
-
-  private
-  def decrypt(encrypted_sk)
-    com.huaweicloud.dis.util.encrypt.EncryptUtils.dec([@encrypt_key].to_java(java.lang.String), encrypted_sk)
-  rescue => e
-    logger.error("Unable to decrypt sk from given configuration",
-                  :decrypt_error_message => e,
-                  :cause => e.respond_to?(:getCause) ? e.getCause() : nil)
-  end
-end #class LogStash::Inputs::Dis