logstash-output-dis 1.1.1 → 1.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c1b97f4e8aa0bb1c00789410e2a4c5ce410a7cb3
-  data.tar.gz: a30635f4d15b0d268acb8802475f67d9dcea9dc1
+  metadata.gz: '019f0e86a61a2053ec8b9dbd53dddda42d57ed37'
+  data.tar.gz: e62e3e0a3158a2fed3856badf0f83c1d465051bb
 SHA512:
-  metadata.gz: '099ec05c009c6c54e0d8c8de13ffdb9042fd47504db2a230af830492c515ab2025f57e79e5cc1a47912a03267afd757ebb7f60db1e014cd7f3906f0cba8fab99'
-  data.tar.gz: 7f2147aa307fca93ec180eb6a5623b6451f50f36901647385f1ceedce85abfed32d94034a84c799ba30c6ed46d4715287b03e683c0280f0270e979f4aac19de9
+  metadata.gz: 0d836921059524dd1a991618aa56e9b676fa15a69d4e98a1175fda13e577e612b2b5e81d32f591cc2708aaaeb3f4897f7af69bbe6caa41b57f9c6423fd4af8ef
+  data.tar.gz: cf32b4f30e9b47eed4c594ee9ffa23fc04ca0b57c9a49a54ba108a799c47990717d49a2bedec392fb58f8fa6c2d38a83e17f08c62c1dd3b02dcebcc4e3640af2

data/Gemfile

@@ -1,11 +1,11 @@
-source 'http://gems.ruby-china.com'
-
-gemspec
-
-logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
-use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
-
-if Dir.exist?(logstash_path) && use_logstash_source
-  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
-  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
-end
+source 'http://gems.ruby-china.com'
+
+gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/LICENSE

@@ -1,13 +1,13 @@
-Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/NOTICE.TXT

@@ -1,5 +1,5 @@
-Elasticsearch
-Copyright 2012-2015 Elasticsearch
-
-This product includes software developed by The Apache Software
+Elasticsearch
+Copyright 2012-2015 Elasticsearch
+
+This product includes software developed by The Apache Software
 Foundation (http://www.apache.org/).

data/README.md

@@ -1,70 +1,70 @@
-# Logstash Output DIS
-
-This is a plugin for [Logstash](https://github.com/elastic/logstash). It will send log records to a DIS stream, using the DIS-Kafka-Adapter.
-
-## Requirements
-
-To get started using this plugin, you will need three things:
-
-1. JDK 1.8 +
-2. JRuby with the Bundler gem installed, 9.0.0.0 +
-3. Maven
-4. Logstash, 6.0.0 to 6.1.0
-
-## Installation
-当前插件未发布到`RubyGems.org`，无法直接从`RubyGems.org`安装插件，只能从本地安装。
-### 0. 修改 RubyGems 镜像地址
-    gem sources --add https://gems.ruby-china.org/ --remove https://rubygems.org/
-
-### 0. 安装 dis-kafka-adapter
-
-
-### 1. 安装 JRuby
-### 2. 安装 Bundler gem
-    gem install bundler
-
-### 3. 安装依赖
-    bundle install
-    rake install_jars
-    gem build logstash-output-dis.gemspec
-
-### 4. 编辑 Logstash 的`Gemfile`，并添加本地插件路径
-    gem "logstash-output-dis", :path => "/your/local/logstash-output-dis"
-
-### 5. 安装插件到 Logstash
-    bin/logstash-plugin install --no-verify
-
-## Usage
-
-```properties
-output
-{
-   dis {
-        stream => ["YOU_DIS_STREAM_NAME"]
-        endpoint => "https://dis.cn-north-1.myhuaweicloud.com"
-        ak => "YOU_ACCESS_KEY_ID"
-        sk => "YOU_SECRET_KEY_ID"
-        region => "cn-north-1"
-        project_id => "YOU_PROJECT_ID"
-        group_id => "YOU_GROUP_ID"
-        decorate_events => true
-        auto_offset_reset => "earliest"
-    }
-}
-```
-
-## Configuration
-
-### Parameters
-
-| Name                     | Description                              | Default                                  |
-| :----------------------- | :--------------------------------------- | :--------------------------------------- |
-| stream                   | 指定在DIS服务上创建的通道名称。             | -                                        |
-| ak                       | 用户的Access Key，可从华为云控制台“我的凭证”页获取。 | -                                        |
-| sk                       | 用户的Secret Key，可从华为云控制台“我的凭证”页获取。 | -                                        |
-| region                   | 将数据上传到指定Region的DIS服务。           | cn-north-1                               |
-| project_id               | 用户所属区域的项目ID，可从华为云控制台“我的凭证”页获取。                    | -                                        |
-| endpoint                 | DIS对应Region的数据接口地址。               | https://dis.cn-north-1.myhuaweicloud.com |
-
-## License
+# Logstash Output DIS
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash). It will send log records to a DIS stream, using the DIS-Kafka-Adapter.
+
+## Requirements
+
+To get started using this plugin, you will need three things:
+
+1. JDK 1.8 +
+2. JRuby with the Bundler gem installed, 9.0.0.0 +
+3. Maven
+4. Logstash, 6.0.0 to 6.1.0
+
+## Installation
+当前插件未发布到`RubyGems.org`，无法直接从`RubyGems.org`安装插件，只能从本地安装。
+### 0. 修改 RubyGems 镜像地址
+    gem sources --add https://gems.ruby-china.org/ --remove https://rubygems.org/
+
+### 0. 安装 dis-kafka-adapter
+
+
+### 1. 安装 JRuby
+### 2. 安装 Bundler gem
+    gem install bundler
+
+### 3. 安装依赖
+    bundle install
+    rake install_jars
+    gem build logstash-output-dis.gemspec
+
+### 4. 编辑 Logstash 的`Gemfile`，并添加本地插件路径
+    gem "logstash-output-dis", :path => "/your/local/logstash-output-dis"
+
+### 5. 安装插件到 Logstash
+    bin/logstash-plugin install --no-verify
+
+## Usage
+
+```properties
+output
+{
+   dis {
+        stream => ["YOU_DIS_STREAM_NAME"]
+        endpoint => "https://dis.cn-north-1.myhuaweicloud.com"
+        ak => "YOU_ACCESS_KEY_ID"
+        sk => "YOU_SECRET_KEY_ID"
+        region => "cn-north-1"
+        project_id => "YOU_PROJECT_ID"
+        group_id => "YOU_GROUP_ID"
+        decorate_events => true
+        auto_offset_reset => "earliest"
+    }
+}
+```
+
+## Configuration
+
+### Parameters
+
+| Name                     | Description                              | Default                                  |
+| :----------------------- | :--------------------------------------- | :--------------------------------------- |
+| stream                   | 指定在DIS服务上创建的通道名称。             | -                                        |
+| ak                       | 用户的Access Key，可从华为云控制台“我的凭证”页获取。 | -                                        |
+| sk                       | 用户的Secret Key，可从华为云控制台“我的凭证”页获取。 | -                                        |
+| region                   | 将数据上传到指定Region的DIS服务。           | cn-north-1                               |
+| project_id               | 用户所属区域的项目ID，可从华为云控制台“我的凭证”页获取。                    | -                                        |
+| endpoint                 | DIS对应Region的数据接口地址。               | https://dis.cn-north-1.myhuaweicloud.com |
+
+## License
 [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0.html)

data/lib/logstash/outputs/dis.rb

@@ -1,318 +1,318 @@
-require 'logstash/namespace'
-require 'logstash/outputs/base'
-require 'java'
-require 'logstash-output-dis_jars.rb'
-
-java_import com.huaweicloud.dis.adapter.kafka.clients.producer.ProducerRecord
-
-# Write events to a DIS stream, using DIS Kafka Adapter.
-class LogStash::Outputs::Dis < LogStash::Outputs::Base
-  declare_threadsafe!
-
-  config_name 'dis'
-
-  default :codec, 'plain'
-
-  config :default_trusted_jks_enabled, :validate => :boolean, :default => false
-  config :security_token, :validate => :string
-  config :exception_retries, :validate => :number, :default => 8
-  config :records_retries, :validate => :number, :default => 20
-  config :proxy_host, :validate => :string
-  config :proxy_port, :validate => :number, :default => 80
-  config :proxy_protocol, :validate => ["http", "https"], :default => "http"
-  config :proxy_username, :validate => :string
-  config :proxy_password, :validate => :string
-  config :proxy_workstation, :validate => :string
-  config :proxy_domain, :validate => :string
-  config :proxy_non_proxy_hosts, :validate => :string
-
-  # The producer will attempt to batch records together into fewer requests whenever multiple
-  # records are being sent to the same partition. This helps performance on both the client
-  # and the server. This configuration controls the default batch size in bytes.
-  config :batch_size, :validate => :number, :default => 16384
-  config :batch_count, :validate => :number, :default => 5000
-
-  # The total bytes of memory the producer can use to buffer records waiting to be sent to the server.
-  config :buffer_memory, :validate => :number, :default => 33554432
-  config :buffer_count, :validate => :number, :default => 5000
-  # The producer groups together any records that arrive in between request
-  # transmissions into a single batched request. Normally this occurs only under
-  # load when records arrive faster than they can be sent out. However in some circumstances
-  # the client may want to reduce the number of requests even under moderate load.
-  # This setting accomplishes this by adding a small amount of artificial delay—that is,
-  # rather than immediately sending out a record the producer will wait for up to the given delay
-  # to allow other records to be sent so that the sends can be batched together.
-  config :linger_ms, :validate => :number, :default => 50
-  config :block_on_buffer_full, :validate => :boolean, :default => false
-  # block time when buffer is full
-  config :max_block_ms, :validate => :number, :default => 60000
-  # max wait time in single backoff
-  config :backoff_max_interval_ms, :validate => :number, :default => 30000
-  config :max_in_flight_requests_per_connection, :validate => :number, :default => 50
-  config :records_retriable_error_code, :validate => :string, :default => "DIS.4303,DIS.5"
-  config :order_by_partition, :validate => :boolean, :default => false
-  config :metadata_timeout_ms, :validate => :number, :default => 600000
-  # The key for the message
-  config :message_key, :validate => :string
-  config :partition_id, :validate => :string
-  # the timeout setting for initial metadata request to fetch topic metadata.
-  config :metadata_fetch_timeout_ms, :validate => :number, :default => 60000
-  # the max time in milliseconds before a metadata refresh is forced.
-  config :metadata_max_age_ms, :validate => :number, :default => 300000
-  # The size of the TCP receive buffer to use when reading data
-  config :receive_buffer_bytes, :validate => :number, :default => 32768
-  # The configuration controls the maximum amount of time the client will wait
-  # for the response of a request. If the response is not received before the timeout
-  # elapses the client will resend the request if necessary or fail the request if
-  # retries are exhausted.
-  config :request_timeout_ms, :validate => :string
-  # The default retry behavior is to retry until successful. To prevent data loss,
-  # the use of this setting is discouraged.
-  #
-  # If you choose to set `retries`, a value greater than zero will cause the
-  # client to only retry a fixed number of times. This will result in data loss
-  # if a transient error outlasts your retry count.
-  #
-  # A value less than zero is a configuration error.
-  config :retries, :validate => :number
-  # The amount of time to wait before attempting to retry a failed produce request to a given topic partition.
-  config :retry_backoff_ms, :validate => :number, :default => 100
-
-
-  # The DIS stream to produce messages to
-  config :stream, :validate => :string, :required => true
-  # DIS Gateway endpoint
-  config :endpoint, :validate => :string, :default => "https://dis.cn-north-1.myhuaweicloud.com"
-  # The ProjectId of the specified region, it can be obtained from My Credential Page
-  config :project_id, :validate => :string
-  # Specifies use which region of DIS, now DIS only support cn-north-1
-  config :region, :validate => :string, :default => "cn-north-1"
-  # The Access Key ID for hwclouds, it can be obtained from My Credential Page
-  config :ak, :validate => :string, :required => true
-  # The Secret key ID is encrypted or not
-  config :is_sk_encrypted, :default => false
-  # The encrypt key used to encypt the Secret Key Id
-  config :encrypt_key, :validate => :string
-  # The Secret Key ID for hwclouds, it can be obtained from My Credential Page
-  config :sk, :validate => :string, :required => true
-  # Serializer class for the key of the message
-  config :key_serializer, :validate => :string, :default => 'com.huaweicloud.dis.adapter.kafka.common.serialization.StringSerializer'
-  # Serializer class for the value of the message
-  config :value_serializer, :validate => :string, :default => 'com.huaweicloud.dis.adapter.kafka.common.serialization.StringSerializer'
-
-  public
-  def register
-    @thread_batch_map = Concurrent::Hash.new
-
-    if !@retries.nil? 
-      if @retries < 0
-        raise ConfigurationError, "A negative retry count (#{@retries}) is not valid. Must be a value >= 0"
-      end
-
-      @logger.warn("Kafka output is configured with finite retry. This instructs Logstash to LOSE DATA after a set number of send attempts fails. If you do not want to lose data if Kafka is down, then you must remove the retry setting.", :retries => @retries)
-    end
-
-
-    @producer = create_producer
-    if value_serializer == 'com.huaweicloud.dis.adapter.kafka.common.serialization.StringSerializer'
-      @codec.on_event do |event, data|
-        write_to_dis(event, data)
-      end
-    elsif value_serializer == 'com.huaweicloud.dis.adapter.kafka.common.serialization.ByteArraySerializer'
-      @codec.on_event do |event, data|
-        write_to_dis(event, data.to_java_bytes)
-      end
-    else
-      raise ConfigurationError, "'value_serializer' only supports com.huaweicloud.dis.adapter.kafka.common.serialization.ByteArraySerializer and com.huaweicloud.dis.adapter.kafka.common.serialization.StringSerializer" 
-    end
-  end
-
-  # def register
-
-  def prepare(record)
-    # This output is threadsafe, so we need to keep a batch per thread.
-    @thread_batch_map[Thread.current].add(record)
-  end
-
-  def multi_receive(events)
-    t = Thread.current
-    if !@thread_batch_map.include?(t)
-      @thread_batch_map[t] = java.util.ArrayList.new(events.size)
-    end
-
-    events.each do |event|
-      break if event == LogStash::SHUTDOWN
-      @codec.encode(event)
-    end
-
-    batch = @thread_batch_map[t]
-    if batch.any?
-      retrying_send(batch)
-      batch.clear
-    end
-  end
-
-  def retrying_send(batch)
-    remaining = @retries;
-
-    while batch.any?
-      if !remaining.nil?
-        if remaining < 0
-          # TODO(sissel): Offer to DLQ? Then again, if it's a transient fault,
-          # DLQing would make things worse (you dlq data that would be successful
-          # after the fault is repaired)
-          logger.info("Exhausted user-configured retry count when sending to Kafka. Dropping these events.",
-                      :max_retries => @retries, :drop_count => batch.count)
-          break
-        end
-
-        remaining -= 1
-      end
-
-      failures = []
-
-      futures = batch.collect do |record| 
-        begin
-          # send() can throw an exception even before the future is created.
-          @producer.send(record)
-        rescue org.apache.kafka.common.errors.TimeoutException => e
-          failures << record
-          nil
-        rescue org.apache.kafka.common.errors.InterruptException => e
-          failures << record
-          nil
-        rescue com.huaweicloud.dis.adapter.kafka.common.errors.SerializationException => e
-          # TODO(sissel): Retrying will fail because the data itself has a problem serializing.
-          # TODO(sissel): Let's add DLQ here.
-          failures << record
-          nil
-        end
-      end.compact
-
-      futures.each_with_index do |future, i|
-        begin
-          result = future.get()
-        rescue => e
-          # TODO(sissel): Add metric to count failures, possibly by exception type.
-          logger.warn("KafkaProducer.send() failed: #{e}", :exception => e)
-          failures << batch[i]
-        end
-      end
-
-      # No failures? Cool. Let's move on.
-      break if failures.empty?
-
-      # Otherwise, retry with any failed transmissions
-      batch = failures
-      delay = @retry_backoff_ms / 1000.0
-      logger.info("Sending batch to DIS failed. Will retry after a delay.", :batch_size => batch.size,
-                  :failures => failures.size, :sleep => delay);
-      sleep(delay)
-    end
-
-  end
-
-  def close
-    @producer.close
-  end
-
-  private
-
-  def write_to_dis(event, serialized_data)
-    stream = event.get("stream");
-	if stream.nil?
-	  stream = @stream;
-	end
-
-	message_key = event.get("partition_key");
-	if message_key.nil?
-	  message_key = @message_key;
-	end
-
-	partition_id = event.get("partition_id");
-
-	if message_key.nil? && partition_id.nil?
-	  # record = ProducerRecord.new(event.sprintf(@stream), serialized_data)
-	  record = ProducerRecord.new(stream, serialized_data)
-	elsif partition_id.nil?
-	  # record = ProducerRecord.new(event.sprintf(@stream), event.sprintf(@message_key), serialized_data)
-	  # record = ProducerRecord.new(stream, event.sprintf(@message_key), serialized_data)
-	  record = ProducerRecord.new(stream, message_key, serialized_data)
-	else
-	  record = ProducerRecord.new(stream, partition_id.to_i, message_key, serialized_data)
-	end
-	prepare(record)
-  rescue LogStash::ShutdownSignal
-    @logger.debug('DIS Kafka producer got shutdown signal')
-  rescue => e
-    @logger.warn('DIS kafka producer threw exception, restarting',
-                 :exception => e)
-  end
-
-  def create_producer
-    begin
-      props = java.util.Properties.new
-      kafka = com.huaweicloud.dis.adapter.kafka.clients.producer.ProducerConfig
-
-      props.put("IS_DEFAULT_TRUSTED_JKS_ENABLED", default_trusted_jks_enabled.to_s)
-      props.put("security.token", security_token) unless security_token.nil?
-      props.put("exception.retries", exception_retries.to_s)
-      props.put("records.retries", records_retries.to_s)
-      props.put("PROXY_HOST", proxy_host) unless proxy_host.nil?
-      props.put("PROXY_PORT", proxy_port.to_s)
-      props.put("PROXY_PROTOCOL", proxy_protocol)
-      props.put("PROXY_USERNAME", proxy_username) unless proxy_username.nil?
-      props.put("PROXY_PASSWORD", proxy_password) unless proxy_password.nil?
-      props.put("PROXY_WORKSTATION", proxy_workstation) unless proxy_workstation.nil?
-      props.put("PROXY_DOMAIN", proxy_domain) unless proxy_domain.nil?
-      props.put("NON_PROXY_HOSTS", proxy_non_proxy_hosts) unless proxy_non_proxy_hosts.nil?
-      
-      props.put("batch.size", batch_size.to_s)
-      props.put("batch.count", batch_count.to_s)
-      props.put("buffer.memory", buffer_memory.to_s)
-      props.put("buffer.count", buffer_count.to_s)
-      props.put("linger.ms", linger_ms.to_s)
-      props.put("block.on.buffer.full", block_on_buffer_full.to_s)
-      props.put("max.block.ms", max_block_ms.to_s)
-      props.put("backoff.max.interval.ms", backoff_max_interval_ms.to_s)
-      props.put("max.in.flight.requests.per.connection", max_in_flight_requests_per_connection.to_s)
-      props.put("records.retriable.error.code", records_retriable_error_code) unless records_retriable_error_code.nil?
-      props.put("order.by.partition", order_by_partition.to_s)
-      props.put("metadata.timeout.ms", metadata_timeout_ms.to_s)
-      # props.put(kafka::RETRIES_CONFIG, retries.to_s) unless retries.nil?
-      # props.put(kafka::RETRY_BACKOFF_MS_CONFIG, retry_backoff_ms.to_s)
-      props.put("key.deserializer", "com.huaweicloud.dis.adapter.kafka.common.serialization.StringDeserializer") 
-      props.put("value.deserializer", "com.huaweicloud.dis.adapter.kafka.common.serialization.StringDeserializer")
-
-      # endpoint, project_id, region, ak, sk
-      props.put("endpoint", endpoint)
-      props.put("projectId", project_id)
-      props.put("region", region)
-      props.put("ak", ak)
-      if is_sk_encrypted
-        decrypted_sk = decrypt(@sk)
-        props.put("sk", decrypted_sk)
-      else
-        props.put("sk", sk)
-      end
-
-
-      com.huaweicloud.dis.adapter.kafka.clients.producer.DISKafkaProducer.new(props)
-    rescue => e
-      logger.error("Unable to create DIS Kafka producer from given configuration",
-                   :kafka_error_message => e,
-                   :cause => e.respond_to?(:getCause) ? e.getCause() : nil)
-      raise e
-    end
-  end
-
-  private
-  def decrypt(encrypted_sk)
-    com.huaweicloud.dis.util.encrypt.EncryptUtils.dec([@encrypt_key].to_java(java.lang.String), encrypted_sk)
-  rescue => e
-    logger.error("Unable to decrypt sk from given configuration",
-                  :decrypt_error_message => e,
-                  :cause => e.respond_to?(:getCause) ? e.getCause() : nil)
-  end
-
-end #class LogStash::Outputs::Dis
+require 'logstash/namespace'
+require 'logstash/outputs/base'
+require 'java'
+require 'logstash-output-dis_jars.rb'
+
+java_import com.huaweicloud.dis.adapter.kafka.clients.producer.ProducerRecord
+
+# Write events to a DIS stream, using DIS Kafka Adapter.
+class LogStash::Outputs::Dis < LogStash::Outputs::Base
+  declare_threadsafe!
+
+  config_name 'dis'
+
+  default :codec, 'plain'
+
+  config :default_trusted_jks_enabled, :validate => :boolean, :default => false
+  config :security_token, :validate => :string
+  config :exception_retries, :validate => :number, :default => 8
+  config :records_retries, :validate => :number, :default => 20
+  config :proxy_host, :validate => :string
+  config :proxy_port, :validate => :number, :default => 80
+  config :proxy_protocol, :validate => ["http", "https"], :default => "http"
+  config :proxy_username, :validate => :string
+  config :proxy_password, :validate => :string
+  config :proxy_workstation, :validate => :string
+  config :proxy_domain, :validate => :string
+  config :proxy_non_proxy_hosts, :validate => :string
+
+  # The producer will attempt to batch records together into fewer requests whenever multiple
+  # records are being sent to the same partition. This helps performance on both the client
+  # and the server. This configuration controls the default batch size in bytes.
+  config :batch_size, :validate => :number, :default => 16384
+  config :batch_count, :validate => :number, :default => 5000
+
+  # The total bytes of memory the producer can use to buffer records waiting to be sent to the server.
+  config :buffer_memory, :validate => :number, :default => 33554432
+  config :buffer_count, :validate => :number, :default => 5000
+  # The producer groups together any records that arrive in between request
+  # transmissions into a single batched request. Normally this occurs only under
+  # load when records arrive faster than they can be sent out. However in some circumstances
+  # the client may want to reduce the number of requests even under moderate load.
+  # This setting accomplishes this by adding a small amount of artificial delay—that is,
+  # rather than immediately sending out a record the producer will wait for up to the given delay
+  # to allow other records to be sent so that the sends can be batched together.
+  config :linger_ms, :validate => :number, :default => 50
+  config :block_on_buffer_full, :validate => :boolean, :default => false
+  # block time when buffer is full
+  config :max_block_ms, :validate => :number, :default => 60000
+  # max wait time in single backoff
+  config :backoff_max_interval_ms, :validate => :number, :default => 30000
+  config :max_in_flight_requests_per_connection, :validate => :number, :default => 50
+  config :records_retriable_error_code, :validate => :string, :default => "DIS.4303,DIS.5"
+  config :order_by_partition, :validate => :boolean, :default => false
+  config :metadata_timeout_ms, :validate => :number, :default => 600000
+  # The key for the message
+  config :message_key, :validate => :string
+  config :partition_id, :validate => :string
+  # the timeout setting for initial metadata request to fetch topic metadata.
+  config :metadata_fetch_timeout_ms, :validate => :number, :default => 60000
+  # the max time in milliseconds before a metadata refresh is forced.
+  config :metadata_max_age_ms, :validate => :number, :default => 300000
+  # The size of the TCP receive buffer to use when reading data
+  config :receive_buffer_bytes, :validate => :number, :default => 32768
+  # The configuration controls the maximum amount of time the client will wait
+  # for the response of a request. If the response is not received before the timeout
+  # elapses the client will resend the request if necessary or fail the request if
+  # retries are exhausted.
+  config :request_timeout_ms, :validate => :string
+  # The default retry behavior is to retry until successful. To prevent data loss,
+  # the use of this setting is discouraged.
+  #
+  # If you choose to set `retries`, a value greater than zero will cause the
+  # client to only retry a fixed number of times. This will result in data loss
+  # if a transient error outlasts your retry count.
+  #
+  # A value less than zero is a configuration error.
+  config :retries, :validate => :number
+  # The amount of time to wait before attempting to retry a failed produce request to a given topic partition.
+  config :retry_backoff_ms, :validate => :number, :default => 100
+
+
+  # The DIS stream to produce messages to
+  config :stream, :validate => :string, :required => true
+  # DIS Gateway endpoint
+  config :endpoint, :validate => :string, :default => "https://dis.cn-north-1.myhuaweicloud.com"
+  # The ProjectId of the specified region, it can be obtained from My Credential Page
+  config :project_id, :validate => :string
+  # Specifies use which region of DIS, now DIS only support cn-north-1
+  config :region, :validate => :string, :default => "cn-north-1"
+  # The Access Key ID for hwclouds, it can be obtained from My Credential Page
+  config :ak, :validate => :string, :required => true
+  # The Secret key ID is encrypted or not
+  config :is_sk_encrypted, :default => false
+  # The encrypt key used to encypt the Secret Key Id
+  config :encrypt_key, :validate => :string
+  # The Secret Key ID for hwclouds, it can be obtained from My Credential Page
+  config :sk, :validate => :string, :required => true
+  # Serializer class for the key of the message
+  config :key_serializer, :validate => :string, :default => 'com.huaweicloud.dis.adapter.kafka.common.serialization.StringSerializer'
+  # Serializer class for the value of the message
+  config :value_serializer, :validate => :string, :default => 'com.huaweicloud.dis.adapter.kafka.common.serialization.StringSerializer'
+
+  public
+  def register
+    @thread_batch_map = Concurrent::Hash.new
+
+    if !@retries.nil? 
+      if @retries < 0
+        raise ConfigurationError, "A negative retry count (#{@retries}) is not valid. Must be a value >= 0"
+      end
+
+      @logger.warn("Kafka output is configured with finite retry. This instructs Logstash to LOSE DATA after a set number of send attempts fails. If you do not want to lose data if Kafka is down, then you must remove the retry setting.", :retries => @retries)
+    end
+
+
+    @producer = create_producer
+    if value_serializer == 'com.huaweicloud.dis.adapter.kafka.common.serialization.StringSerializer'
+      @codec.on_event do |event, data|
+        write_to_dis(event, data)
+      end
+    elsif value_serializer == 'com.huaweicloud.dis.adapter.kafka.common.serialization.ByteArraySerializer'
+      @codec.on_event do |event, data|
+        write_to_dis(event, data.to_java_bytes)
+      end
+    else
+      raise ConfigurationError, "'value_serializer' only supports com.huaweicloud.dis.adapter.kafka.common.serialization.ByteArraySerializer and com.huaweicloud.dis.adapter.kafka.common.serialization.StringSerializer" 
+    end
+  end
+
+  # def register
+
+  def prepare(record)
+    # This output is threadsafe, so we need to keep a batch per thread.
+    @thread_batch_map[Thread.current].add(record)
+  end
+
+  def multi_receive(events)
+    t = Thread.current
+    if !@thread_batch_map.include?(t)
+      @thread_batch_map[t] = java.util.ArrayList.new(events.size)
+    end
+
+    events.each do |event|
+      break if event == LogStash::SHUTDOWN
+      @codec.encode(event)
+    end
+
+    batch = @thread_batch_map[t]
+    if batch.any?
+      retrying_send(batch)
+      batch.clear
+    end
+  end
+
+  def retrying_send(batch)
+    remaining = @retries;
+
+    while batch.any?
+      if !remaining.nil?
+        if remaining < 0
+          # TODO(sissel): Offer to DLQ? Then again, if it's a transient fault,
+          # DLQing would make things worse (you dlq data that would be successful
+          # after the fault is repaired)
+          logger.info("Exhausted user-configured retry count when sending to Kafka. Dropping these events.",
+                      :max_retries => @retries, :drop_count => batch.count)
+          break
+        end
+
+        remaining -= 1
+      end
+
+      failures = []
+
+      futures = batch.collect do |record| 
+        begin
+          # send() can throw an exception even before the future is created.
+          @producer.send(record)
+        rescue org.apache.kafka.common.errors.TimeoutException => e
+          failures << record
+          nil
+        rescue org.apache.kafka.common.errors.InterruptException => e
+          failures << record
+          nil
+        rescue com.huaweicloud.dis.adapter.kafka.common.errors.SerializationException => e
+          # TODO(sissel): Retrying will fail because the data itself has a problem serializing.
+          # TODO(sissel): Let's add DLQ here.
+          failures << record
+          nil
+        end
+      end.compact
+
+      futures.each_with_index do |future, i|
+        begin
+          result = future.get()
+        rescue => e
+          # TODO(sissel): Add metric to count failures, possibly by exception type.
+          logger.warn("KafkaProducer.send() failed: #{e}", :exception => e)
+          failures << batch[i]
+        end
+      end
+
+      # No failures? Cool. Let's move on.
+      break if failures.empty?
+
+      # Otherwise, retry with any failed transmissions
+      batch = failures
+      delay = @retry_backoff_ms / 1000.0
+      logger.info("Sending batch to DIS failed. Will retry after a delay.", :batch_size => batch.size,
+                  :failures => failures.size, :sleep => delay);
+      sleep(delay)
+    end
+
+  end
+
+  def close
+    @producer.close
+  end
+
+  private
+
+  def write_to_dis(event, serialized_data)
+    stream = event.get("stream");
+	if stream.nil?
+	  stream = @stream;
+	end
+
+	message_key = event.get("partition_key");
+	if message_key.nil?
+	  message_key = @message_key;
+	end
+
+	partition_id = event.get("partition_id");
+
+	if message_key.nil? && partition_id.nil?
+	  # record = ProducerRecord.new(event.sprintf(@stream), serialized_data)
+	  record = ProducerRecord.new(stream, serialized_data)
+	elsif partition_id.nil?
+	  # record = ProducerRecord.new(event.sprintf(@stream), event.sprintf(@message_key), serialized_data)
+	  # record = ProducerRecord.new(stream, event.sprintf(@message_key), serialized_data)
+	  record = ProducerRecord.new(stream, message_key, serialized_data)
+	else
+	  record = ProducerRecord.new(stream, partition_id.to_i, message_key, serialized_data)
+	end
+	prepare(record)
+  rescue LogStash::ShutdownSignal
+    @logger.debug('DIS Kafka producer got shutdown signal')
+  rescue => e
+    @logger.warn('DIS kafka producer threw exception, restarting',
+                 :exception => e)
+  end
+
+  def create_producer
+    begin
+      props = java.util.Properties.new
+      kafka = com.huaweicloud.dis.adapter.kafka.clients.producer.ProducerConfig
+
+      props.put("IS_DEFAULT_TRUSTED_JKS_ENABLED", default_trusted_jks_enabled.to_s)
+      props.put("security.token", security_token) unless security_token.nil?
+      props.put("exception.retries", exception_retries.to_s)
+      props.put("records.retries", records_retries.to_s)
+      props.put("PROXY_HOST", proxy_host) unless proxy_host.nil?
+      props.put("PROXY_PORT", proxy_port.to_s)
+      props.put("PROXY_PROTOCOL", proxy_protocol)
+      props.put("PROXY_USERNAME", proxy_username) unless proxy_username.nil?
+      props.put("PROXY_PASSWORD", proxy_password) unless proxy_password.nil?
+      props.put("PROXY_WORKSTATION", proxy_workstation) unless proxy_workstation.nil?
+      props.put("PROXY_DOMAIN", proxy_domain) unless proxy_domain.nil?
+      props.put("NON_PROXY_HOSTS", proxy_non_proxy_hosts) unless proxy_non_proxy_hosts.nil?
+      
+      props.put("batch.size", batch_size.to_s)
+      props.put("batch.count", batch_count.to_s)
+      props.put("buffer.memory", buffer_memory.to_s)
+      props.put("buffer.count", buffer_count.to_s)
+      props.put("linger.ms", linger_ms.to_s)
+      props.put("block.on.buffer.full", block_on_buffer_full.to_s)
+      props.put("max.block.ms", max_block_ms.to_s)
+      props.put("backoff.max.interval.ms", backoff_max_interval_ms.to_s)
+      props.put("max.in.flight.requests.per.connection", max_in_flight_requests_per_connection.to_s)
+      props.put("records.retriable.error.code", records_retriable_error_code) unless records_retriable_error_code.nil?
+      props.put("order.by.partition", order_by_partition.to_s)
+      props.put("metadata.timeout.ms", metadata_timeout_ms.to_s)
+      # props.put(kafka::RETRIES_CONFIG, retries.to_s) unless retries.nil?
+      # props.put(kafka::RETRY_BACKOFF_MS_CONFIG, retry_backoff_ms.to_s)
+      props.put("key.deserializer", "com.huaweicloud.dis.adapter.kafka.common.serialization.StringDeserializer") 
+      props.put("value.deserializer", "com.huaweicloud.dis.adapter.kafka.common.serialization.StringDeserializer")
+
+      # endpoint, project_id, region, ak, sk
+      props.put("endpoint", endpoint)
+      props.put("projectId", project_id)
+      props.put("region", region)
+      props.put("ak", ak)
+      if is_sk_encrypted
+        decrypted_sk = decrypt(@sk)
+        props.put("sk", decrypted_sk)
+      else
+        props.put("sk", sk)
+      end
+
+
+      com.huaweicloud.dis.adapter.kafka.clients.producer.DISKafkaProducer.new(props)
+    rescue => e
+      logger.error("Unable to create DIS Kafka producer from given configuration",
+                   :kafka_error_message => e,
+                   :cause => e.respond_to?(:getCause) ? e.getCause() : nil)
+      raise e
+    end
+  end
+
+  private
+  def decrypt(encrypted_sk)
+    com.huaweicloud.dis.util.encrypt.EncryptUtils.dec([@encrypt_key].to_java(java.lang.String), encrypted_sk)
+  rescue => e
+    logger.error("Unable to decrypt sk from given configuration",
+                  :decrypt_error_message => e,
+                  :cause => e.respond_to?(:getCause) ? e.getCause() : nil)
+  end
+
+end #class LogStash::Outputs::Dis

data/lib/logstash-output-dis_jars.rb

@@ -1,5 +1,5 @@
-# encoding: utf-8
-require 'logstash/environment'
-
-root_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
-LogStash::Environment.load_runtime_jars! File.join(root_dir, "vendor")
+# encoding: utf-8
+require 'logstash/environment'
+
+root_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+LogStash::Environment.load_runtime_jars! File.join(root_dir, "vendor")

data/logstash-output-dis.gemspec

@@ -1,36 +1,36 @@
-Gem::Specification.new do |s|
-
-  s.name            = 'logstash-output-dis'
-  s.version         = '1.1.1'
-  s.licenses        = ['Apache License (2.0)']
-  s.summary         = "Writes events to a DIS stream"
-  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
-  s.authors         = ['Data Ingestion Service']
-  s.email           = 'dis@huaweicloud.com'
-  s.homepage        = "https://www.huaweicloud.com/product/dis.html"
-  s.require_paths = ['lib', 'vendor/jar-dependencies']
-
-  # Files
-  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
-
-  # Tests
-  s.test_files = s.files.grep(%r{^(test|spec|features)/})
-
-  # Special flag to let us know this is actually a logstash plugin
-  s.metadata = { 'logstash_plugin' => 'true', 'group' => 'output'}
-
-  s.requirements << "jar 'com.huaweicloud.dis:huaweicloud-dis-kafka-adapter', '1.2.1'"
-  s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.21'"
-  s.requirements << "jar 'org.apache.logging.log4j:log4j-1.2-api', '2.6.2'"
-
-  s.add_development_dependency 'jar-dependencies', '~> 0.3.2'
-
-  # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency 'logstash-codec-plain'
-  s.add_runtime_dependency 'logstash-codec-json'
-
-  s.add_development_dependency 'logstash-devutils'
-  s.add_development_dependency 'poseidon'
-  s.add_development_dependency 'snappy'
-end
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-output-dis'
+  s.version         = '1.1.2'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "Writes events to a DIS stream"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ['Data Ingestion Service']
+  s.email           = 'dis@huaweicloud.com'
+  s.homepage        = "https://www.huaweicloud.com/product/dis.html"
+  s.require_paths = ['lib', 'vendor/jar-dependencies']
+
+  # Files
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { 'logstash_plugin' => 'true', 'group' => 'output'}
+
+  s.requirements << "jar 'com.huaweicloud.dis:huaweicloud-dis-kafka-adapter', '1.2.1'"
+  s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.21'"
+  s.requirements << "jar 'org.apache.logging.log4j:log4j-1.2-api', '2.6.2'"
+
+  s.add_development_dependency 'jar-dependencies', '~> 0.3.2'
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'logstash-codec-json'
+
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'poseidon'
+  s.add_development_dependency 'snappy'
+end

data/spec/unit/outputs/dis_spec.rb

@@ -1,128 +1,128 @@
-# encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require 'logstash/outputs/dis'
-require 'json'
-
-describe "outputs/dis" do
-  let (:simple_dis_config) {{'stream' => 'test', 'project_id' => 'test_project_id', 'ak' => 'test_ak', 'sk' => 'test_sk'}}
-  let (:event) { LogStash::Event.new({'message' => 'hello', 'stream_name' => 'my_stream', 'host' => '127.0.0.1',
-                                      '@timestamp' => LogStash::Timestamp.now}) }
-
-  context 'when initializing' do
-    it "should register" do
-      output = LogStash::Plugin.lookup("output", "dis").new(simple_dis_config)
-      expect {output.register}.to_not raise_error
-    end
-
-    it 'should populate dis config with default values' do
-      dis = LogStash::Outputs::Dis.new(simple_dis_config)
-      insist {dis.endpoint} == 'https://dis.cn-north-1.myhuaweicloud.com'
-      insist {dis.stream} == 'test'
-      insist {dis.key_serializer} == 'com.huaweicloud.dis.adapter.kafka.common.serialization.StringSerializer'
-    end
-  end
-
-  context 'when outputting messages' do
-    #it 'should send logstash event to DIS' do
-      #expect_any_instance_of(com.huaweicloud.dis.adapter.kafka.clients.producer.DISKafkaProducer).to receive(:send)
-        #.with(an_instance_of(com.huaweicloud.dis.adapter.kafka.clients.producer.ProducerRecord)).and_call_original
-      #dis = LogStash::Outputs::Dis.new(simple_dis_config)
-      #dis.register
-      #dis.multi_receive([event])
-    #end
-
-    #it 'should support field referenced message_keys' do
-      #expect(com.huaweicloud.dis.adapter.kafka.clients.producer.ProducerRecord).to receive(:new)
-        #.with("test", "127.0.0.1", event.to_s).and_call_original
-      #expect_any_instance_of(com.huaweicloud.dis.adapter.kafka.clients.producer.DISKafkaProducer).to receive(:send).and_call_original
-      #dis = LogStash::Outputs::Dis.new(simple_dis_config.merge({"message_key" => "%{host}"}))
-      #dis.register
-      #dis.multi_receive([event])
-    #end
-  end
-  
-  context "when DISKafkaProducer#send() raises an exception" do
-    let(:failcount) { (rand * 10).to_i }
-    let(:sendcount) { failcount + 1 }
-
-    let(:exception_classes) { [
-      com.huaweicloud.dis.adapter.kafka.common.errors.TimeoutException,
-      com.huaweicloud.dis.adapter.kafka.common.errors.InterruptException,
-      com.huaweicloud.dis.adapter.kafka.common.errors.SerializationException 
-    ] }
-
-    before do
-      count = 0
-      expect_any_instance_of(com.huaweicloud.dis.adapter.kafka.clients.producer.DISKafkaProducer).to receive(:send)
-        .exactly(sendcount).times
-        .and_wrap_original do |m, *args|
-        if count < failcount # fail 'failcount' times in a row.
-          count += 1
-          # Pick an exception at random
-          raise exception_classes.shuffle.first.new("injected exception for testing")
-        else
-          #m.call(*args) # call original
-        end
-      end
-    end
-
-    it "should retry until successful" do
-      dis = LogStash::Outputs::Dis.new(simple_dis_config)
-      dis.register
-      dis.multi_receive([event])
-    end
-  end
-
-  context "when a send fails" do
-    context "and the default retries behavior is used" do
-      # Fail this many times and then finally succeed.
-      let(:failcount) { (rand * 10).to_i }
-
-      # Expect DISKafkaProducer.send() to get called again after every failure, plus the successful one.
-      let(:sendcount) { failcount + 1 }
-
-
-      it "should retry until successful" do
-        count = 0;
-
-        expect_any_instance_of(com.huaweicloud.dis.adapter.kafka.clients.producer.DISKafkaProducer).to receive(:send)
-              .exactly(sendcount).times
-              .and_wrap_original do |m, *args|
-          if count < failcount
-            count += 1
-            # inject some failures.
-
-            # Return a custom Future that will raise an exception to simulate a DIS send() problem.
-            future = java.util.concurrent.FutureTask.new { raise "Failed" }
-            future.run
-            future
-          else
-            #m.call(*args)
-          end
-        end
-        dis = LogStash::Outputs::Dis.new(simple_dis_config)
-        dis.register
-        dis.multi_receive([event])
-      end
-    end
-
-    context "and when retries is set by the user" do
-      let(:retries) { (rand * 10).to_i }
-      let(:max_sends) { retries + 1 }
-
-      it "should give up after retries are exhausted" do
-        expect_any_instance_of(com.huaweicloud.dis.adapter.kafka.clients.producer.DISKafkaProducer).to receive(:send)
-              .at_most(max_sends).times
-              .and_wrap_original do |m, *args|
-          # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
-          future.run
-          future
-        end
-        dis = LogStash::Outputs::Dis.new(simple_dis_config.merge("retries" => retries))
-        dis.register
-        dis.multi_receive([event])
-      end
-    end
-  end
-end
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require 'logstash/outputs/dis'
+require 'json'
+
+describe "outputs/dis" do
+  let (:simple_dis_config) {{'stream' => 'test', 'project_id' => 'test_project_id', 'ak' => 'test_ak', 'sk' => 'test_sk'}}
+  let (:event) { LogStash::Event.new({'message' => 'hello', 'stream_name' => 'my_stream', 'host' => '127.0.0.1',
+                                      '@timestamp' => LogStash::Timestamp.now}) }
+
+  context 'when initializing' do
+    it "should register" do
+      output = LogStash::Plugin.lookup("output", "dis").new(simple_dis_config)
+      expect {output.register}.to_not raise_error
+    end
+
+    it 'should populate dis config with default values' do
+      dis = LogStash::Outputs::Dis.new(simple_dis_config)
+      insist {dis.endpoint} == 'https://dis.cn-north-1.myhuaweicloud.com'
+      insist {dis.stream} == 'test'
+      insist {dis.key_serializer} == 'com.huaweicloud.dis.adapter.kafka.common.serialization.StringSerializer'
+    end
+  end
+
+  context 'when outputting messages' do
+    #it 'should send logstash event to DIS' do
+      #expect_any_instance_of(com.huaweicloud.dis.adapter.kafka.clients.producer.DISKafkaProducer).to receive(:send)
+        #.with(an_instance_of(com.huaweicloud.dis.adapter.kafka.clients.producer.ProducerRecord)).and_call_original
+      #dis = LogStash::Outputs::Dis.new(simple_dis_config)
+      #dis.register
+      #dis.multi_receive([event])
+    #end
+
+    #it 'should support field referenced message_keys' do
+      #expect(com.huaweicloud.dis.adapter.kafka.clients.producer.ProducerRecord).to receive(:new)
+        #.with("test", "127.0.0.1", event.to_s).and_call_original
+      #expect_any_instance_of(com.huaweicloud.dis.adapter.kafka.clients.producer.DISKafkaProducer).to receive(:send).and_call_original
+      #dis = LogStash::Outputs::Dis.new(simple_dis_config.merge({"message_key" => "%{host}"}))
+      #dis.register
+      #dis.multi_receive([event])
+    #end
+  end
+  
+  context "when DISKafkaProducer#send() raises an exception" do
+    let(:failcount) { (rand * 10).to_i }
+    let(:sendcount) { failcount + 1 }
+
+    let(:exception_classes) { [
+      com.huaweicloud.dis.adapter.kafka.common.errors.TimeoutException,
+      com.huaweicloud.dis.adapter.kafka.common.errors.InterruptException,
+      com.huaweicloud.dis.adapter.kafka.common.errors.SerializationException 
+    ] }
+
+    before do
+      count = 0
+      expect_any_instance_of(com.huaweicloud.dis.adapter.kafka.clients.producer.DISKafkaProducer).to receive(:send)
+        .exactly(sendcount).times
+        .and_wrap_original do |m, *args|
+        if count < failcount # fail 'failcount' times in a row.
+          count += 1
+          # Pick an exception at random
+          raise exception_classes.shuffle.first.new("injected exception for testing")
+        else
+          #m.call(*args) # call original
+        end
+      end
+    end
+
+    it "should retry until successful" do
+      dis = LogStash::Outputs::Dis.new(simple_dis_config)
+      dis.register
+      dis.multi_receive([event])
+    end
+  end
+
+  context "when a send fails" do
+    context "and the default retries behavior is used" do
+      # Fail this many times and then finally succeed.
+      let(:failcount) { (rand * 10).to_i }
+
+      # Expect DISKafkaProducer.send() to get called again after every failure, plus the successful one.
+      let(:sendcount) { failcount + 1 }
+
+
+      it "should retry until successful" do
+        count = 0;
+
+        expect_any_instance_of(com.huaweicloud.dis.adapter.kafka.clients.producer.DISKafkaProducer).to receive(:send)
+              .exactly(sendcount).times
+              .and_wrap_original do |m, *args|
+          if count < failcount
+            count += 1
+            # inject some failures.
+
+            # Return a custom Future that will raise an exception to simulate a DIS send() problem.
+            future = java.util.concurrent.FutureTask.new { raise "Failed" }
+            future.run
+            future
+          else
+            #m.call(*args)
+          end
+        end
+        dis = LogStash::Outputs::Dis.new(simple_dis_config)
+        dis.register
+        dis.multi_receive([event])
+      end
+    end
+
+    context "and when retries is set by the user" do
+      let(:retries) { (rand * 10).to_i }
+      let(:max_sends) { retries + 1 }
+
+      it "should give up after retries are exhausted" do
+        expect_any_instance_of(com.huaweicloud.dis.adapter.kafka.clients.producer.DISKafkaProducer).to receive(:send)
+              .at_most(max_sends).times
+              .and_wrap_original do |m, *args|
+          # Always fail.
+          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future.run
+          future
+        end
+        dis = LogStash::Outputs::Dis.new(simple_dis_config.merge("retries" => retries))
+        dis.register
+        dis.multi_receive([event])
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-dis
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.1.2
 platform: ruby
 authors:
 - Data Ingestion Service
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-04-23 00:00:00.000000000 Z
+date: 2019-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement