logstash-output-oss 0.1.1-java

data/spec/integration/common.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/logging/logger'
+require 'logstash/outputs/oss'
+require 'stud/temporary'
+java_import 'com.aliyun.oss.model.GetObjectRequest'
+
+# This file contains the common logic used by integration tests
+shared_context "plugin initialize" do
+  let(:endpoint) { ENV['OSS_ENDPOINT'] }
+  let(:bucket) { ENV['OSS_BUCKET'] }
+  let(:access_key_id) { ENV['OSS_ACCESS_KEY'] }
+  let(:access_key_secret) { ENV['OSS_SECRET_KEY'] }
+  let(:prefix) { "logstash" }
+  let(:temporary_directory) { Stud::Temporary.pathname }
+
+  let(:common_configurations) do
+    {
+      "endpoint" => endpoint,
+      "bucket" => bucket,
+      "access_key_id" => access_key_id,
+      "access_key_secret" => access_key_secret,
+      "prefix" => prefix,
+      "size_rotate" => 1024,
+      "time_rotate" => 60,
+      "temporary_directory" => temporary_directory,
+    }
+  end
+
+  LogStash::Logging::Logger::configure_logging("debug") if ENV["DEBUG"]
+
+  let(:oss) { OSSClientBuilder.new().build(endpoint, access_key_id, access_key_secret) }
+  subject { LogStash::Outputs::OSS.new(configurations) }
+end
+
+# remove object with `prefix`
+def clean_bucket
+  oss.listObjects(bucket, prefix).getObjectSummaries().each do |objectSummary|
+    oss.deleteObject(bucket, objectSummary.getKey())
+  end
+end

data/spec/integration/encoding_spec.rb

@@ -0,0 +1,92 @@
+# encoding: utf-8
+
+require 'rspec'
+require_relative 'common'
+
+describe 'encoding integration tests', :integration => true do
+  include_context 'plugin initialize'
+
+  let(:number_of_events) { 500 }
+  let(:batch_size) { 125 }
+  let(:event_encoded) { "Hello world\n" }
+  let(:batch) do
+    b = {}
+    number_of_events.times do
+      event = LogStash::Event.new({ "message" => event_encoded, "host" => "localhost", "index" => 1 })
+      b[event] = "#{event_encoded}"
+    end
+    b
+  end
+
+  before do
+    subject.register
+
+    batch.each_slice(batch_size) do |smaller_batch|
+      subject.multi_receive_encoded(smaller_batch)
+      sleep(1)
+    end
+
+    subject.close
+  end
+
+  after do
+    clean_bucket
+  end
+
+  ################### gzip encoding integration test ###################
+  describe 'gzip encoding integration tests' do
+    let(:configurations) {
+      common_configurations.merge(
+        {
+          "encoding" => "gzip",
+          "rotation_strategy" => "time",
+          "time_rotate" => 0.01 # 0.01 * 60 = 0.6s
+        }
+      )
+    }
+
+    it 'compare file content(uncompressed) with original events' do
+      # read lines from oss bucket
+      copy_file_to_temporary_dir(temporary_directory)
+
+      expect(Dir.glob(File.join(temporary_directory, prefix, "*.gz")).inject(0) { |sum, f| sum + Zlib::GzipReader.new(File.open(f)).readlines.size }).to eq(number_of_events)
+    end
+
+    it 'creates multiples files' do
+      expect(oss.listObjects(bucket, prefix).getObjectSummaries().size).to be_between(3, 4)
+    end
+  end
+
+  ################### plain encoding integration test ###################
+  describe 'plain encoding integration test' do
+    let(:configurations) {
+      common_configurations.merge(
+        {
+          "encoding" => "none",
+          "rotation_strategy" => "time",
+          "time_rotate" => 0.01 # 0.01 * 60 = 0.6s
+        }
+      )
+    }
+
+    it 'compare file content with original events' do
+      # read lines from oss bucket
+      copy_file_to_temporary_dir(temporary_directory)
+
+      expect(Dir.glob(File.join(temporary_directory, prefix, "*.data")).inject(0) { |sum, f| sum + File.open(f).readlines.size }).to eq(number_of_events)
+    end
+
+    it 'creates multiples files' do
+      expect(oss.listObjects(bucket, prefix).getObjectSummaries().size).to be_between(3, 4)
+    end
+  end
+
+  def copy_file_to_temporary_dir(temporary_directory)
+    FileUtils.rm_rf(temporary_directory)
+    FileUtils.mkdir_p(File.join(temporary_directory, prefix))
+    oss.listObjects(bucket, prefix).getObjectSummaries().each do |objectSummary|
+      request = GetObjectRequest.new(bucket, objectSummary.getKey())
+      oss.getObject(request, java.io.File.new(File.join(temporary_directory, objectSummary.getKey())))
+    end
+  end
+end

data/spec/integration/oss_spec.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+
+require 'rspec'
+require_relative 'common'
+
+describe 'integration tests', :integration => true do
+
+  include_context 'plugin initialize'
+
+  let(:number_of_events) { 400000 }
+  let(:batch_size) { 400 }
+  let(:event_encoded) { "Hello world\n" }
+  let(:batch) do
+    b = {}
+    number_of_events.times do
+      event = LogStash::Event.new({ "message" => event_encoded, "host" => "localhost", "index" => Random.rand(3) })
+      b[event] = "#{event_encoded}"
+    end
+    b
+  end
+
+  let(:configurations) {
+    common_configurations.merge(
+      {
+        "prefix" => "logstash/%{index}",
+        "size_rotate" => 13631488, # 13MB
+        "time_rotate" => 0.15 # 0.15 * 60 = 9s
+      }
+    )
+  }
+
+  before do
+    subject.register
+  end
+
+  after do
+    subject.close
+
+    clean_bucket
+  end
+
+  it 'integration stress tests' do
+    100.times do
+      batch.each_slice(batch_size) { |smaller_batch| subject.multi_receive_encoded(smaller_batch) }
+    end
+  end
+end

data/spec/integration/recover_spec.rb

@@ -0,0 +1,43 @@
+# encoding: utf-8
+
+require 'rspec'
+require_relative 'common'
+
+describe 'recover integration tests', :integration => true do
+
+  include_context 'plugin initialize'
+
+  let(:configurations) { common_configurations.merge({}) }
+
+  before do
+    clean_bucket
+  end
+
+  it 'recover files from temporary directory' do
+    generate_files(3)
+    expect(oss.listObjects(bucket, prefix).getObjectSummaries().size).to eq(3)
+  end
+
+  it 'recover files from empty temporary directory' do
+    generate_files(0)
+    expect(oss.listObjects(bucket, prefix).getObjectSummaries().size).to eq(0)
+  end
+
+  after do
+    clean_bucket
+  end
+
+  def generate_files(number_of_files)
+    # generate files in temporary directory
+    generator = LogStash::Outputs::OSS::FileGenerator.new(prefix, "gzip", temporary_directory)
+    number_of_files.times do
+      generator.current_file.write("Hello, world")
+      generator.current_file.close
+      generator.rotate
+    end
+
+    subject.register
+
+    subject.close
+  end
+end

data/spec/outputs/oss/generator_spec.rb

@@ -0,0 +1,67 @@
+# encoding: utf-8
+
+require 'rspec'
+require "logstash/devutils/rspec/spec_helper"
+require 'logstash/outputs/oss'
+require 'logstash/logging/logger'
+
+describe 'file generator unit tests' do
+
+  let (:temporary_directory) { Stud::Temporary.pathname }
+  let (:prefix) { "logstash" }
+
+  describe 'gzip encoding generator' do
+
+    it 'gzip' do
+      generator = LogStash::Outputs::OSS::FileGenerator.new(prefix, "gzip", temporary_directory)
+
+      # part-0, ends with part-0.gz
+      elements = generator.current_file.path.to_s.split(::File::SEPARATOR)
+      file_name = elements.at(elements.size - 1)
+      expect(file_name.match(/ls.oss.*.part-0.gz/).to_s).to eq(file_name)
+
+      generator.rotate
+
+      # part-1, ends with part-1.gz
+      generator.current_file
+      elements = generator.current_file.path.to_s.split(::File::SEPARATOR)
+      file_name = elements.at(elements.size - 1)
+      expect(file_name.match(/ls.oss.*.part-1.gz/).to_s).to eq(file_name)
+
+      generator.rotate
+
+      # part-2, ends with part-2.gz
+      generator.current_file
+      elements = generator.current_file.path.to_s.split(::File::SEPARATOR)
+      file_name = elements.at(elements.size - 1)
+      expect(file_name.match(/ls.oss.*.part-2.gz/).to_s).to eq(file_name)
+    end
+  end
+
+  describe 'plain encoding generator' do
+
+    it 'plain' do
+      generator = LogStash::Outputs::OSS::FileGenerator.new(prefix, "none", temporary_directory)
+      # part-0, ends with part-0.data
+      elements = generator.current_file.path.to_s.split(::File::SEPARATOR)
+      file_name = elements.at(elements.size - 1)
+      expect(file_name.match(/ls.oss.*.part-0.data/).to_s).to eq(file_name)
+
+      generator.rotate
+
+      # part-1, ends with part-1.data
+      generator.current_file
+      elements = generator.current_file.path.to_s.split(::File::SEPARATOR)
+      file_name = elements.at(elements.size - 1)
+      expect(file_name.match(/ls.oss.*.part-1.data/).to_s).to eq(file_name)
+
+      generator.rotate
+
+      # part-2, ends with part-2.data
+      generator.current_file
+      elements = generator.current_file.path.to_s.split(::File::SEPARATOR)
+      file_name = elements.at(elements.size - 1)
+      expect(file_name.match(/ls.oss.*.part-2.data/).to_s).to eq(file_name)
+    end
+  end
+end

data/spec/outputs/oss/rotation_spec.rb

@@ -0,0 +1,69 @@
+# encoding: utf-8
+
+require 'rspec'
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/logging/logger'
+require 'logstash/outputs/oss'
+
+describe 'rotation unit tests' do
+
+  let (:temporary_directory) { Stud::Temporary.pathname }
+  let (:prefix) { "logstash" }
+  let (:encoding) { "none" }
+  let(:event) { "Hello world\n" }
+
+  it 'size based rotation unit tests' do
+    size_based_rotation = LogStash::Outputs::OSS::SizeBasedRotation.new(1024)
+    expect(size_based_rotation.needs_periodic_check?).to be(false)
+
+    generator = LogStash::Outputs::OSS::FileGenerator.new(prefix, encoding, temporary_directory)
+
+    generator.current_file.write(event)
+    expect(size_based_rotation.rotate?(generator.current_file)).to be(false)
+
+    1024.times do
+      generator.current_file.write(event)
+    end
+
+    expect(size_based_rotation.rotate?(generator.current_file)).to be(true)
+
+    generator.current_file.write(event)
+
+    expect(size_based_rotation.rotate?(generator.current_file)).to be(true)
+  end
+
+  it 'time based rotation unit tests' do
+    time_based_rotation = LogStash::Outputs::OSS::TimeBasedRotation.new(0.05)
+    expect(time_based_rotation.needs_periodic_check?).to be(true)
+
+    generator = LogStash::Outputs::OSS::FileGenerator.new(prefix, encoding, temporary_directory)
+
+    generator.current_file.write(event)
+    expect(time_based_rotation.rotate?(generator.current_file)).to be(false)
+
+    sleep(1)
+
+    expect(time_based_rotation.rotate?(generator.current_file)).to be(false)
+
+    sleep(2)
+
+    expect(time_based_rotation.rotate?(generator.current_file)).to be(true)
+  end
+
+  it 'hybrid rotation unit tests' do
+    hybrid_rotation = LogStash::Outputs::OSS::HybridRotation.new(1024, 0.05)
+    expect(hybrid_rotation.needs_periodic_check?).to be(true)
+
+    generator = LogStash::Outputs::OSS::FileGenerator.new(prefix, encoding, temporary_directory)
+
+    generator.current_file.write(event)
+
+    expect(hybrid_rotation.rotate?(generator.current_file)).to be(false)
+
+    1024.times do
+      generator.current_file.write(event)
+    end
+
+    expect(hybrid_rotation.rotate?(generator.current_file)).to be(true)
+  end
+end

data/spec/outputs/oss_spec.rb

@@ -0,0 +1,58 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/oss"
+require "logstash/codecs/plain"
+require "logstash/event"
+
+describe LogStash::Outputs::OSS do
+  let(:options) {
+    {
+        "endpoint" => "oss-cn-zhangjiakou.aliyuncs.com",
+        "bucket" => "bucket",
+        "prefix" => "logstash/%{index}",
+        "recover" => true,
+        "access_key_id" => "access_key_id",
+        "access_key_secret" => "access_key_secret",
+        "encoding" => "none"
+    }
+  }
+
+  describe 'invalid configurations' do
+    it 'validate rotate' do
+      oss = described_class.new(options.merge({ "size_rotate" => -1, "time_rotate" => -1 }))
+      expect { oss.register }.to raise_error(LogStash::ConfigurationError)
+
+      oss = described_class.new(options.merge({ "size_rotate" => -1 }))
+      expect { oss.register }.to raise_error(LogStash::ConfigurationError)
+
+      oss = described_class.new(options.merge({ "time_rotate" => -1 }))
+      expect { oss.register }.to raise_error(LogStash::ConfigurationError)
+    end
+
+
+    it 'validate upload configurations' do
+      oss = described_class.new(options.merge({ "upload_workers_count" => -1, "upload_queue_size" => -1 }))
+      expect { oss.register }.to raise_error(LogStash::ConfigurationError)
+
+      oss = described_class.new(options.merge({ "upload_workers_count" => -1 }))
+      expect { oss.register }.to raise_error(LogStash::ConfigurationError)
+
+      oss = described_class.new(options.merge({ "upload_queue_size" => -1 }))
+      expect { oss.register }.to raise_error(LogStash::ConfigurationError)
+    end
+
+    it 'validate invalid temporary directory' do
+      dir = 'a_' + Time.now.to_i.to_s
+      file_name = dir + '/1.gz'
+      system('mkdir -p ' + dir + ' && touch ' + file_name)
+      oss = described_class.new(options.merge({ "temporary_directory" => file_name }))
+      expect { oss.register }.to raise_error(LogStash::ConfigurationError)
+      system('rm -rf ' + dir)
+    end
+
+    it 'validate invalid additional oss configurations' do
+      oss = described_class.new(options.merge({ "additional_oss_settings" => { "max_connections_to_oss" => -1} }))
+      expect { oss.register }.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+end

metadata

@@ -0,0 +1,206 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-oss
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: java
+authors:
+- jinhu.wu
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2019-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: concurrent-ruby
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.9
+  name: uuid
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.9
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  name: logstash-codec-line
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  name: jar-dependencies
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gem-name. This
+  gem is not a stand-alone program
+email: jinhu.wu.nju@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- DEVELOPER.md
+- Gemfile
+- LICENSE
+- README.md
+- lib/com/aliyun/aliyun-java-sdk-core/3.4.0/aliyun-java-sdk-core-3.4.0.jar
+- lib/com/aliyun/aliyun-java-sdk-ecs/4.2.0/aliyun-java-sdk-ecs-4.2.0.jar
+- lib/com/aliyun/aliyun-java-sdk-ram/3.0.0/aliyun-java-sdk-ram-3.0.0.jar
+- lib/com/aliyun/aliyun-java-sdk-sts/3.0.0/aliyun-java-sdk-sts-3.0.0.jar
+- lib/com/aliyun/oss/aliyun-sdk-oss/3.4.0/aliyun-sdk-oss-3.4.0.jar
+- lib/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar
+- lib/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar
+- lib/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar
+- lib/commons-codec/commons-codec/1.9/commons-codec-1.9.jar
+- lib/commons-logging/commons-logging/1.2/commons-logging-1.2.jar
+- lib/javax/activation/activation/1.1/activation-1.1.jar
+- lib/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar
+- lib/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar
+- lib/logstash-output-oss_jars.rb
+- lib/logstash/outputs/oss.rb
+- lib/logstash/outputs/oss/file_generator.rb
+- lib/logstash/outputs/oss/file_manager.rb
+- lib/logstash/outputs/oss/file_uploader.rb
+- lib/logstash/outputs/oss/gzip_file.rb
+- lib/logstash/outputs/oss/rotations/hybrid_rotation.rb
+- lib/logstash/outputs/oss/rotations/size_based_rotation.rb
+- lib/logstash/outputs/oss/rotations/time_based_rotation.rb
+- lib/logstash/outputs/oss/temporary_file.rb
+- lib/logstash/outputs/oss/version.rb
+- lib/org/apache/httpcomponents/httpclient/4.4.1/httpclient-4.4.1.jar
+- lib/org/apache/httpcomponents/httpcore/4.4.1/httpcore-4.4.1.jar
+- lib/org/codehaus/jackson/jackson-core-asl/1.8.3/jackson-core-asl-1.8.3.jar
+- lib/org/codehaus/jackson/jackson-jaxrs/1.8.3/jackson-jaxrs-1.8.3.jar
+- lib/org/codehaus/jackson/jackson-mapper-asl/1.8.3/jackson-mapper-asl-1.8.3.jar
+- lib/org/codehaus/jackson/jackson-xc/1.8.3/jackson-xc-1.8.3.jar
+- lib/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar
+- lib/org/jdom/jdom/1.1/jdom-1.1.jar
+- lib/stax/stax-api/1.0.1/stax-api-1.0.1.jar
+- logstash-output-oss.gemspec
+- spec/integration/common.rb
+- spec/integration/encoding_spec.rb
+- spec/integration/oss_spec.rb
+- spec/integration/recover_spec.rb
+- spec/outputs/oss/generator_spec.rb
+- spec/outputs/oss/rotation_spec.rb
+- spec/outputs/oss_spec.rb
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- jar com.aliyun.oss:aliyun-sdk-oss, 3.4.0
+rubyforge_project:
+rubygems_version: 2.7.6
+signing_key:
+specification_version: 4
+summary: Sends Logstash events to the Aliyun Object Storage Service
+test_files:
+- spec/integration/common.rb
+- spec/integration/encoding_spec.rb
+- spec/integration/oss_spec.rb
+- spec/integration/recover_spec.rb
+- spec/outputs/oss/generator_spec.rb
+- spec/outputs/oss/rotation_spec.rb
+- spec/outputs/oss_spec.rb