logstash-output-dis 1.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c1b97f4e8aa0bb1c00789410e2a4c5ce410a7cb3
+  data.tar.gz: a30635f4d15b0d268acb8802475f67d9dcea9dc1
+SHA512:
+  metadata.gz: '099ec05c009c6c54e0d8c8de13ffdb9042fd47504db2a230af830492c515ab2025f57e79e5cc1a47912a03267afd757ebb7f60db1e014cd7f3906f0cba8fab99'
+  data.tar.gz: 7f2147aa307fca93ec180eb6a5623b6451f50f36901647385f1ceedce85abfed32d94034a84c799ba30c6ed46d4715287b03e683c0280f0270e979f4aac19de9

data/Gemfile

@@ -0,0 +1,11 @@
+source 'http://gems.ruby-china.com'
+
+gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/NOTICE.TXT

@@ -0,0 +1,5 @@
+Elasticsearch
+Copyright 2012-2015 Elasticsearch
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).

data/README.md

@@ -0,0 +1,70 @@
+# Logstash Output DIS
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash). It will send log records to a DIS stream, using the DIS-Kafka-Adapter.
+
+## Requirements
+
+To get started using this plugin, you will need three things:
+
+1. JDK 1.8 +
+2. JRuby with the Bundler gem installed, 9.0.0.0 +
+3. Maven
+4. Logstash, 6.0.0 to 6.1.0
+
+## Installation
+当前插件未发布到`RubyGems.org`，无法直接从`RubyGems.org`安装插件，只能从本地安装。
+### 0. 修改 RubyGems 镜像地址
+    gem sources --add https://gems.ruby-china.org/ --remove https://rubygems.org/
+
+### 0. 安装 dis-kafka-adapter
+
+
+### 1. 安装 JRuby
+### 2. 安装 Bundler gem
+    gem install bundler
+
+### 3. 安装依赖
+    bundle install
+    rake install_jars
+    gem build logstash-output-dis.gemspec
+
+### 4. 编辑 Logstash 的`Gemfile`，并添加本地插件路径
+    gem "logstash-output-dis", :path => "/your/local/logstash-output-dis"
+
+### 5. 安装插件到 Logstash
+    bin/logstash-plugin install --no-verify
+
+## Usage
+
+```properties
+output
+{
+   dis {
+        stream => ["YOU_DIS_STREAM_NAME"]
+        endpoint => "https://dis.cn-north-1.myhuaweicloud.com"
+        ak => "YOU_ACCESS_KEY_ID"
+        sk => "YOU_SECRET_KEY_ID"
+        region => "cn-north-1"
+        project_id => "YOU_PROJECT_ID"
+        group_id => "YOU_GROUP_ID"
+        decorate_events => true
+        auto_offset_reset => "earliest"
+    }
+}
+```
+
+## Configuration
+
+### Parameters
+
+| Name                     | Description                              | Default                                  |
+| :----------------------- | :--------------------------------------- | :--------------------------------------- |
+| stream                   | 指定在DIS服务上创建的通道名称。             | -                                        |
+| ak                       | 用户的Access Key，可从华为云控制台“我的凭证”页获取。 | -                                        |
+| sk                       | 用户的Secret Key，可从华为云控制台“我的凭证”页获取。 | -                                        |
+| region                   | 将数据上传到指定Region的DIS服务。           | cn-north-1                               |
+| project_id               | 用户所属区域的项目ID，可从华为云控制台“我的凭证”页获取。                    | -                                        |
+| endpoint                 | DIS对应Region的数据接口地址。               | https://dis.cn-north-1.myhuaweicloud.com |
+
+## License
+[Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0.html)

data/lib/logstash/outputs/dis.rb

@@ -0,0 +1,318 @@
+require 'logstash/namespace'
+require 'logstash/outputs/base'
+require 'java'
+require 'logstash-output-dis_jars.rb'
+
+java_import com.huaweicloud.dis.adapter.kafka.clients.producer.ProducerRecord
+
+# Write events to a DIS stream, using DIS Kafka Adapter.
+class LogStash::Outputs::Dis < LogStash::Outputs::Base
+  declare_threadsafe!
+
+  config_name 'dis'
+
+  default :codec, 'plain'
+
+  config :default_trusted_jks_enabled, :validate => :boolean, :default => false
+  config :security_token, :validate => :string
+  config :exception_retries, :validate => :number, :default => 8
+  config :records_retries, :validate => :number, :default => 20
+  config :proxy_host, :validate => :string
+  config :proxy_port, :validate => :number, :default => 80
+  config :proxy_protocol, :validate => ["http", "https"], :default => "http"
+  config :proxy_username, :validate => :string
+  config :proxy_password, :validate => :string
+  config :proxy_workstation, :validate => :string
+  config :proxy_domain, :validate => :string
+  config :proxy_non_proxy_hosts, :validate => :string
+
+  # The producer will attempt to batch records together into fewer requests whenever multiple
+  # records are being sent to the same partition. This helps performance on both the client
+  # and the server. This configuration controls the default batch size in bytes.
+  config :batch_size, :validate => :number, :default => 16384
+  config :batch_count, :validate => :number, :default => 5000
+
+  # The total bytes of memory the producer can use to buffer records waiting to be sent to the server.
+  config :buffer_memory, :validate => :number, :default => 33554432
+  config :buffer_count, :validate => :number, :default => 5000
+  # The producer groups together any records that arrive in between request
+  # transmissions into a single batched request. Normally this occurs only under
+  # load when records arrive faster than they can be sent out. However in some circumstances
+  # the client may want to reduce the number of requests even under moderate load.
+  # This setting accomplishes this by adding a small amount of artificial delay—that is,
+  # rather than immediately sending out a record the producer will wait for up to the given delay
+  # to allow other records to be sent so that the sends can be batched together.
+  config :linger_ms, :validate => :number, :default => 50
+  config :block_on_buffer_full, :validate => :boolean, :default => false
+  # block time when buffer is full
+  config :max_block_ms, :validate => :number, :default => 60000
+  # max wait time in single backoff
+  config :backoff_max_interval_ms, :validate => :number, :default => 30000
+  config :max_in_flight_requests_per_connection, :validate => :number, :default => 50
+  config :records_retriable_error_code, :validate => :string, :default => "DIS.4303,DIS.5"
+  config :order_by_partition, :validate => :boolean, :default => false
+  config :metadata_timeout_ms, :validate => :number, :default => 600000
+  # The key for the message
+  config :message_key, :validate => :string
+  config :partition_id, :validate => :string
+  # the timeout setting for initial metadata request to fetch topic metadata.
+  config :metadata_fetch_timeout_ms, :validate => :number, :default => 60000
+  # the max time in milliseconds before a metadata refresh is forced.
+  config :metadata_max_age_ms, :validate => :number, :default => 300000
+  # The size of the TCP receive buffer to use when reading data
+  config :receive_buffer_bytes, :validate => :number, :default => 32768
+  # The configuration controls the maximum amount of time the client will wait
+  # for the response of a request. If the response is not received before the timeout
+  # elapses the client will resend the request if necessary or fail the request if
+  # retries are exhausted.
+  config :request_timeout_ms, :validate => :string
+  # The default retry behavior is to retry until successful. To prevent data loss,
+  # the use of this setting is discouraged.
+  #
+  # If you choose to set `retries`, a value greater than zero will cause the
+  # client to only retry a fixed number of times. This will result in data loss
+  # if a transient error outlasts your retry count.
+  #
+  # A value less than zero is a configuration error.
+  config :retries, :validate => :number
+  # The amount of time to wait before attempting to retry a failed produce request to a given topic partition.
+  config :retry_backoff_ms, :validate => :number, :default => 100
+
+
+  # The DIS stream to produce messages to
+  config :stream, :validate => :string, :required => true
+  # DIS Gateway endpoint
+  config :endpoint, :validate => :string, :default => "https://dis.cn-north-1.myhuaweicloud.com"
+  # The ProjectId of the specified region, it can be obtained from My Credential Page
+  config :project_id, :validate => :string
+  # Specifies use which region of DIS, now DIS only support cn-north-1
+  config :region, :validate => :string, :default => "cn-north-1"
+  # The Access Key ID for hwclouds, it can be obtained from My Credential Page
+  config :ak, :validate => :string, :required => true
+  # The Secret key ID is encrypted or not
+  config :is_sk_encrypted, :default => false
+  # The encrypt key used to encypt the Secret Key Id
+  config :encrypt_key, :validate => :string
+  # The Secret Key ID for hwclouds, it can be obtained from My Credential Page
+  config :sk, :validate => :string, :required => true
+  # Serializer class for the key of the message
+  config :key_serializer, :validate => :string, :default => 'com.huaweicloud.dis.adapter.kafka.common.serialization.StringSerializer'
+  # Serializer class for the value of the message
+  config :value_serializer, :validate => :string, :default => 'com.huaweicloud.dis.adapter.kafka.common.serialization.StringSerializer'
+
+  public
+  def register
+    @thread_batch_map = Concurrent::Hash.new
+
+    if !@retries.nil? 
+      if @retries < 0
+        raise ConfigurationError, "A negative retry count (#{@retries}) is not valid. Must be a value >= 0"
+      end
+
+      @logger.warn("Kafka output is configured with finite retry. This instructs Logstash to LOSE DATA after a set number of send attempts fails. If you do not want to lose data if Kafka is down, then you must remove the retry setting.", :retries => @retries)
+    end
+
+
+    @producer = create_producer
+    if value_serializer == 'com.huaweicloud.dis.adapter.kafka.common.serialization.StringSerializer'
+      @codec.on_event do |event, data|
+        write_to_dis(event, data)
+      end
+    elsif value_serializer == 'com.huaweicloud.dis.adapter.kafka.common.serialization.ByteArraySerializer'
+      @codec.on_event do |event, data|
+        write_to_dis(event, data.to_java_bytes)
+      end
+    else
+      raise ConfigurationError, "'value_serializer' only supports com.huaweicloud.dis.adapter.kafka.common.serialization.ByteArraySerializer and com.huaweicloud.dis.adapter.kafka.common.serialization.StringSerializer" 
+    end
+  end
+
+  # def register
+
+  def prepare(record)
+    # This output is threadsafe, so we need to keep a batch per thread.
+    @thread_batch_map[Thread.current].add(record)
+  end
+
+  def multi_receive(events)
+    t = Thread.current
+    if !@thread_batch_map.include?(t)
+      @thread_batch_map[t] = java.util.ArrayList.new(events.size)
+    end
+
+    events.each do |event|
+      break if event == LogStash::SHUTDOWN
+      @codec.encode(event)
+    end
+
+    batch = @thread_batch_map[t]
+    if batch.any?
+      retrying_send(batch)
+      batch.clear
+    end
+  end
+
+  def retrying_send(batch)
+    remaining = @retries;
+
+    while batch.any?
+      if !remaining.nil?
+        if remaining < 0
+          # TODO(sissel): Offer to DLQ? Then again, if it's a transient fault,
+          # DLQing would make things worse (you dlq data that would be successful
+          # after the fault is repaired)
+          logger.info("Exhausted user-configured retry count when sending to Kafka. Dropping these events.",
+                      :max_retries => @retries, :drop_count => batch.count)
+          break
+        end
+
+        remaining -= 1
+      end
+
+      failures = []
+
+      futures = batch.collect do |record| 
+        begin
+          # send() can throw an exception even before the future is created.
+          @producer.send(record)
+        rescue org.apache.kafka.common.errors.TimeoutException => e
+          failures << record
+          nil
+        rescue org.apache.kafka.common.errors.InterruptException => e
+          failures << record
+          nil
+        rescue com.huaweicloud.dis.adapter.kafka.common.errors.SerializationException => e
+          # TODO(sissel): Retrying will fail because the data itself has a problem serializing.
+          # TODO(sissel): Let's add DLQ here.
+          failures << record
+          nil
+        end
+      end.compact
+
+      futures.each_with_index do |future, i|
+        begin
+          result = future.get()
+        rescue => e
+          # TODO(sissel): Add metric to count failures, possibly by exception type.
+          logger.warn("KafkaProducer.send() failed: #{e}", :exception => e)
+          failures << batch[i]
+        end
+      end
+
+      # No failures? Cool. Let's move on.
+      break if failures.empty?
+
+      # Otherwise, retry with any failed transmissions
+      batch = failures
+      delay = @retry_backoff_ms / 1000.0
+      logger.info("Sending batch to DIS failed. Will retry after a delay.", :batch_size => batch.size,
+                  :failures => failures.size, :sleep => delay);
+      sleep(delay)
+    end
+
+  end
+
+  def close
+    @producer.close
+  end
+
+  private
+
+  def write_to_dis(event, serialized_data)
+    stream = event.get("stream");
+	if stream.nil?
+	  stream = @stream;
+	end
+
+	message_key = event.get("partition_key");
+	if message_key.nil?
+	  message_key = @message_key;
+	end
+
+	partition_id = event.get("partition_id");
+
+	if message_key.nil? && partition_id.nil?
+	  # record = ProducerRecord.new(event.sprintf(@stream), serialized_data)
+	  record = ProducerRecord.new(stream, serialized_data)
+	elsif partition_id.nil?
+	  # record = ProducerRecord.new(event.sprintf(@stream), event.sprintf(@message_key), serialized_data)
+	  # record = ProducerRecord.new(stream, event.sprintf(@message_key), serialized_data)
+	  record = ProducerRecord.new(stream, message_key, serialized_data)
+	else
+	  record = ProducerRecord.new(stream, partition_id.to_i, message_key, serialized_data)
+	end
+	prepare(record)
+  rescue LogStash::ShutdownSignal
+    @logger.debug('DIS Kafka producer got shutdown signal')
+  rescue => e
+    @logger.warn('DIS kafka producer threw exception, restarting',
+                 :exception => e)
+  end
+
+  def create_producer
+    begin
+      props = java.util.Properties.new
+      kafka = com.huaweicloud.dis.adapter.kafka.clients.producer.ProducerConfig
+
+      props.put("IS_DEFAULT_TRUSTED_JKS_ENABLED", default_trusted_jks_enabled.to_s)
+      props.put("security.token", security_token) unless security_token.nil?
+      props.put("exception.retries", exception_retries.to_s)
+      props.put("records.retries", records_retries.to_s)
+      props.put("PROXY_HOST", proxy_host) unless proxy_host.nil?
+      props.put("PROXY_PORT", proxy_port.to_s)
+      props.put("PROXY_PROTOCOL", proxy_protocol)
+      props.put("PROXY_USERNAME", proxy_username) unless proxy_username.nil?
+      props.put("PROXY_PASSWORD", proxy_password) unless proxy_password.nil?
+      props.put("PROXY_WORKSTATION", proxy_workstation) unless proxy_workstation.nil?
+      props.put("PROXY_DOMAIN", proxy_domain) unless proxy_domain.nil?
+      props.put("NON_PROXY_HOSTS", proxy_non_proxy_hosts) unless proxy_non_proxy_hosts.nil?
+      
+      props.put("batch.size", batch_size.to_s)
+      props.put("batch.count", batch_count.to_s)
+      props.put("buffer.memory", buffer_memory.to_s)
+      props.put("buffer.count", buffer_count.to_s)
+      props.put("linger.ms", linger_ms.to_s)
+      props.put("block.on.buffer.full", block_on_buffer_full.to_s)
+      props.put("max.block.ms", max_block_ms.to_s)
+      props.put("backoff.max.interval.ms", backoff_max_interval_ms.to_s)
+      props.put("max.in.flight.requests.per.connection", max_in_flight_requests_per_connection.to_s)
+      props.put("records.retriable.error.code", records_retriable_error_code) unless records_retriable_error_code.nil?
+      props.put("order.by.partition", order_by_partition.to_s)
+      props.put("metadata.timeout.ms", metadata_timeout_ms.to_s)
+      # props.put(kafka::RETRIES_CONFIG, retries.to_s) unless retries.nil?
+      # props.put(kafka::RETRY_BACKOFF_MS_CONFIG, retry_backoff_ms.to_s)
+      props.put("key.deserializer", "com.huaweicloud.dis.adapter.kafka.common.serialization.StringDeserializer") 
+      props.put("value.deserializer", "com.huaweicloud.dis.adapter.kafka.common.serialization.StringDeserializer")
+
+      # endpoint, project_id, region, ak, sk
+      props.put("endpoint", endpoint)
+      props.put("projectId", project_id)
+      props.put("region", region)
+      props.put("ak", ak)
+      if is_sk_encrypted
+        decrypted_sk = decrypt(@sk)
+        props.put("sk", decrypted_sk)
+      else
+        props.put("sk", sk)
+      end
+
+
+      com.huaweicloud.dis.adapter.kafka.clients.producer.DISKafkaProducer.new(props)
+    rescue => e
+      logger.error("Unable to create DIS Kafka producer from given configuration",
+                   :kafka_error_message => e,
+                   :cause => e.respond_to?(:getCause) ? e.getCause() : nil)
+      raise e
+    end
+  end
+
+  private
+  def decrypt(encrypted_sk)
+    com.huaweicloud.dis.util.encrypt.EncryptUtils.dec([@encrypt_key].to_java(java.lang.String), encrypted_sk)
+  rescue => e
+    logger.error("Unable to decrypt sk from given configuration",
+                  :decrypt_error_message => e,
+                  :cause => e.respond_to?(:getCause) ? e.getCause() : nil)
+  end
+
+end #class LogStash::Outputs::Dis

data/lib/logstash-output-dis_jars.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+require 'logstash/environment'
+
+root_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+LogStash::Environment.load_runtime_jars! File.join(root_dir, "vendor")

data/logstash-output-dis.gemspec

@@ -0,0 +1,36 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-output-dis'
+  s.version         = '1.1.1'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "Writes events to a DIS stream"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ['Data Ingestion Service']
+  s.email           = 'dis@huaweicloud.com'
+  s.homepage        = "https://www.huaweicloud.com/product/dis.html"
+  s.require_paths = ['lib', 'vendor/jar-dependencies']
+
+  # Files
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { 'logstash_plugin' => 'true', 'group' => 'output'}
+
+  s.requirements << "jar 'com.huaweicloud.dis:huaweicloud-dis-kafka-adapter', '1.2.1'"
+  s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.21'"
+  s.requirements << "jar 'org.apache.logging.log4j:log4j-1.2-api', '2.6.2'"
+
+  s.add_development_dependency 'jar-dependencies', '~> 0.3.2'
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'logstash-codec-json'
+
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'poseidon'
+  s.add_development_dependency 'snappy'
+end

data/spec/unit/outputs/dis_spec.rb

@@ -0,0 +1,128 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require 'logstash/outputs/dis'
+require 'json'
+
+describe "outputs/dis" do
+  let (:simple_dis_config) {{'stream' => 'test', 'project_id' => 'test_project_id', 'ak' => 'test_ak', 'sk' => 'test_sk'}}
+  let (:event) { LogStash::Event.new({'message' => 'hello', 'stream_name' => 'my_stream', 'host' => '127.0.0.1',
+                                      '@timestamp' => LogStash::Timestamp.now}) }
+
+  context 'when initializing' do
+    it "should register" do
+      output = LogStash::Plugin.lookup("output", "dis").new(simple_dis_config)
+      expect {output.register}.to_not raise_error
+    end
+
+    it 'should populate dis config with default values' do
+      dis = LogStash::Outputs::Dis.new(simple_dis_config)
+      insist {dis.endpoint} == 'https://dis.cn-north-1.myhuaweicloud.com'
+      insist {dis.stream} == 'test'
+      insist {dis.key_serializer} == 'com.huaweicloud.dis.adapter.kafka.common.serialization.StringSerializer'
+    end
+  end
+
+  context 'when outputting messages' do
+    #it 'should send logstash event to DIS' do
+      #expect_any_instance_of(com.huaweicloud.dis.adapter.kafka.clients.producer.DISKafkaProducer).to receive(:send)
+        #.with(an_instance_of(com.huaweicloud.dis.adapter.kafka.clients.producer.ProducerRecord)).and_call_original
+      #dis = LogStash::Outputs::Dis.new(simple_dis_config)
+      #dis.register
+      #dis.multi_receive([event])
+    #end
+
+    #it 'should support field referenced message_keys' do
+      #expect(com.huaweicloud.dis.adapter.kafka.clients.producer.ProducerRecord).to receive(:new)
+        #.with("test", "127.0.0.1", event.to_s).and_call_original
+      #expect_any_instance_of(com.huaweicloud.dis.adapter.kafka.clients.producer.DISKafkaProducer).to receive(:send).and_call_original
+      #dis = LogStash::Outputs::Dis.new(simple_dis_config.merge({"message_key" => "%{host}"}))
+      #dis.register
+      #dis.multi_receive([event])
+    #end
+  end
+  
+  context "when DISKafkaProducer#send() raises an exception" do
+    let(:failcount) { (rand * 10).to_i }
+    let(:sendcount) { failcount + 1 }
+
+    let(:exception_classes) { [
+      com.huaweicloud.dis.adapter.kafka.common.errors.TimeoutException,
+      com.huaweicloud.dis.adapter.kafka.common.errors.InterruptException,
+      com.huaweicloud.dis.adapter.kafka.common.errors.SerializationException 
+    ] }
+
+    before do
+      count = 0
+      expect_any_instance_of(com.huaweicloud.dis.adapter.kafka.clients.producer.DISKafkaProducer).to receive(:send)
+        .exactly(sendcount).times
+        .and_wrap_original do |m, *args|
+        if count < failcount # fail 'failcount' times in a row.
+          count += 1
+          # Pick an exception at random
+          raise exception_classes.shuffle.first.new("injected exception for testing")
+        else
+          #m.call(*args) # call original
+        end
+      end
+    end
+
+    it "should retry until successful" do
+      dis = LogStash::Outputs::Dis.new(simple_dis_config)
+      dis.register
+      dis.multi_receive([event])
+    end
+  end
+
+  context "when a send fails" do
+    context "and the default retries behavior is used" do
+      # Fail this many times and then finally succeed.
+      let(:failcount) { (rand * 10).to_i }
+
+      # Expect DISKafkaProducer.send() to get called again after every failure, plus the successful one.
+      let(:sendcount) { failcount + 1 }
+
+
+      it "should retry until successful" do
+        count = 0;
+
+        expect_any_instance_of(com.huaweicloud.dis.adapter.kafka.clients.producer.DISKafkaProducer).to receive(:send)
+              .exactly(sendcount).times
+              .and_wrap_original do |m, *args|
+          if count < failcount
+            count += 1
+            # inject some failures.
+
+            # Return a custom Future that will raise an exception to simulate a DIS send() problem.
+            future = java.util.concurrent.FutureTask.new { raise "Failed" }
+            future.run
+            future
+          else
+            #m.call(*args)
+          end
+        end
+        dis = LogStash::Outputs::Dis.new(simple_dis_config)
+        dis.register
+        dis.multi_receive([event])
+      end
+    end
+
+    context "and when retries is set by the user" do
+      let(:retries) { (rand * 10).to_i }
+      let(:max_sends) { retries + 1 }
+
+      it "should give up after retries are exhausted" do
+        expect_any_instance_of(com.huaweicloud.dis.adapter.kafka.clients.producer.DISKafkaProducer).to receive(:send)
+              .at_most(max_sends).times
+              .and_wrap_original do |m, *args|
+          # Always fail.
+          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future.run
+          future
+        end
+        dis = LogStash::Outputs::Dis.new(simple_dis_config.merge("retries" => retries))
+        dis.register
+        dis.multi_receive([event])
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,212 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-dis
+version: !ruby/object:Gem::Version
+  version: 1.1.1
+platform: ruby
+authors:
+- Data Ingestion Service
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2019-04-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.2
+  name: jar-dependencies
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.2
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: poseidon
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: snappy
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
+email: dis@huaweicloud.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- NOTICE.TXT
+- README.md
+- lib/com/fasterxml/jackson/core/jackson-annotations/2.8.11/jackson-annotations-2.8.11.jar
+- lib/com/fasterxml/jackson/core/jackson-core/2.8.11/jackson-core-2.8.11.jar
+- lib/com/fasterxml/jackson/core/jackson-databind/2.8.11.3/jackson-databind-2.8.11.3.jar
+- lib/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar
+- lib/com/huaweicloud/dis/huaweicloud-dis-kafka-adapter-common/1.2.1/huaweicloud-dis-kafka-adapter-common-1.2.1.jar
+- lib/com/huaweicloud/dis/huaweicloud-dis-kafka-adapter/1.2.1/huaweicloud-dis-kafka-adapter-1.2.1.jar
+- lib/com/huaweicloud/dis/huaweicloud-sdk-java-dis-iface/1.3.3/huaweicloud-sdk-java-dis-iface-1.3.3.jar
+- lib/com/huaweicloud/dis/huaweicloud-sdk-java-dis/1.3.3/huaweicloud-sdk-java-dis-1.3.3.jar
+- lib/commons-codec/commons-codec/1.9/commons-codec-1.9.jar
+- lib/commons-io/commons-io/2.4/commons-io-2.4.jar
+- lib/commons-logging/commons-logging/1.2/commons-logging-1.2.jar
+- lib/joda-time/joda-time/2.8.1/joda-time-2.8.1.jar
+- lib/log4j/log4j/1.2.17/log4j-1.2.17.jar
+- lib/logstash-output-dis_jars.rb
+- lib/logstash/outputs/dis.rb
+- lib/org/apache/httpcomponents/httpasyncclient/4.1.3/httpasyncclient-4.1.3.jar
+- lib/org/apache/httpcomponents/httpclient/4.5.2/httpclient-4.5.2.jar
+- lib/org/apache/httpcomponents/httpcore-nio/4.4.6/httpcore-nio-4.4.6.jar
+- lib/org/apache/httpcomponents/httpcore/4.4.4/httpcore-4.4.4.jar
+- lib/org/apache/httpcomponents/httpmime/4.5.2/httpmime-4.5.2.jar
+- lib/org/apache/logging/log4j/log4j-1.2-api/2.6.2/log4j-1.2-api-2.6.2.jar
+- lib/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar
+- lib/org/apache/logging/log4j/log4j-core/2.6.2/log4j-core-2.6.2.jar
+- lib/org/slf4j/slf4j-api/1.7.21/slf4j-api-1.7.21.jar
+- lib/org/slf4j/slf4j-log4j12/1.7.21/slf4j-log4j12-1.7.21.jar
+- lib/org/xerial/snappy/snappy-java/1.1.7.2/snappy-java-1.1.7.2.jar
+- logstash-output-dis.gemspec
+- spec/unit/outputs/dis_spec.rb
+- vendor/jar-dependencies/runtime-jars/commons-codec-1.9.jar
+- vendor/jar-dependencies/runtime-jars/commons-io-2.4.jar
+- vendor/jar-dependencies/runtime-jars/commons-logging-1.2.jar
+- vendor/jar-dependencies/runtime-jars/httpasyncclient-4.1.3.jar
+- vendor/jar-dependencies/runtime-jars/httpclient-4.5.2.jar
+- vendor/jar-dependencies/runtime-jars/httpcore-4.4.4.jar
+- vendor/jar-dependencies/runtime-jars/httpcore-nio-4.4.6.jar
+- vendor/jar-dependencies/runtime-jars/httpmime-4.5.2.jar
+- vendor/jar-dependencies/runtime-jars/huaweicloud-dis-kafka-adapter-1.2.1.jar
+- vendor/jar-dependencies/runtime-jars/huaweicloud-dis-kafka-adapter-common-1.2.1.jar
+- vendor/jar-dependencies/runtime-jars/huaweicloud-sdk-java-dis-1.3.3.jar
+- vendor/jar-dependencies/runtime-jars/huaweicloud-sdk-java-dis-iface-1.3.3.jar
+- vendor/jar-dependencies/runtime-jars/jackson-annotations-2.8.11.jar
+- vendor/jar-dependencies/runtime-jars/jackson-core-2.8.11.jar
+- vendor/jar-dependencies/runtime-jars/jackson-databind-2.8.11.3.jar
+- vendor/jar-dependencies/runtime-jars/joda-time-2.8.1.jar
+- vendor/jar-dependencies/runtime-jars/log4j-1.2-api-2.6.2.jar
+- vendor/jar-dependencies/runtime-jars/log4j-1.2.17.jar
+- vendor/jar-dependencies/runtime-jars/log4j-api-2.6.2.jar
+- vendor/jar-dependencies/runtime-jars/log4j-core-2.6.2.jar
+- vendor/jar-dependencies/runtime-jars/protobuf-java-2.5.0.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.21.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-log4j12-1.7.21.jar
+- vendor/jar-dependencies/runtime-jars/snappy-java-1.1.7.2.jar
+homepage: https://www.huaweicloud.com/product/dis.html
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+- vendor/jar-dependencies
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- jar 'com.huaweicloud.dis:huaweicloud-dis-kafka-adapter', '1.2.1'
+- jar 'org.slf4j:slf4j-log4j12', '1.7.21'
+- jar 'org.apache.logging.log4j:log4j-1.2-api', '2.6.2'
+rubyforge_project:
+rubygems_version: 2.6.14.1
+signing_key:
+specification_version: 4
+summary: Writes events to a DIS stream
+test_files:
+- spec/unit/outputs/dis_spec.rb