logstash-output-s3 3.2.0 → 4.0.0

data/spec/spec_helper.rb

@@ -0,0 +1,6 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require_relative "supports/helpers"
+require "logstash/logging/logger"
+
+LogStash::Logging::Logger::configure_logging("debug") if ENV["DEBUG"]

data/spec/supports/helpers.rb

@@ -1,14 +1,38 @@
-def delete_matching_keys_on_bucket(prefix)
-  s3_object.buckets[minimal_settings["bucket"]].objects.with_prefix(prefix).each do |obj|
-    obj.delete
+# encoding: utf-8
+shared_context "setup plugin" do
+  let(:temporary_directory) { Stud::Temporary.pathname }
+
+  let(:bucket) { ENV["AWS_LOGSTASH_TEST_BUCKET"] }
+  let(:access_key_id) {  ENV["AWS_ACCESS_KEY_ID"] }
+  let(:secret_access_key) { ENV["AWS_SECRET_ACCESS_KEY"] }
+  let(:size_file) { 100 }
+  let(:time_file) { 100 }
+  let(:tags) { [] }
+  let(:prefix) { "home" }
+  let(:region) { "us-east-1" }
+
+  let(:main_options) do
+    {
+      "bucket" => bucket,
+      "prefix" => prefix,
+      "temporary_directory" => temporary_directory,
+      "access_key_id" => access_key_id,
+      "secret_access_key" => secret_access_key,
+      "size_file" => size_file,
+      "time_file" => time_file,
+      "region" => region,
+      "tags" => []
+    }
   end
-end
 
-def key_exists_on_bucket?(key)
-  s3_object.buckets[minimal_settings["bucket"]].objects[key].exists?
-end
+  let(:client_credentials) { Aws::Credentials.new(access_key_id, secret_access_key) }
+  let(:bucket_resource) { Aws::S3::Bucket.new(bucket, { :credentials => client_credentials, :region => region }) }
 
-def events_in_files(files)
-  files.collect { |file| File.foreach(file).count }.inject(&:+)
+  subject { LogStash::Outputs::S3.new(options) }
 end
 
+def clean_remote_files(prefix = "")
+  bucket_resource.objects(:prefix => prefix).each do |object|
+    object.delete
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-s3
 version: !ruby/object:Gem::Version
-  version: 3.2.0
+  version: 4.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-09-12 00:00:00.000000000 Z
+date: 2016-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -44,6 +44,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: concurrent-ruby
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -114,9 +128,38 @@ files:
 - NOTICE.TXT
 - README.md
 - lib/logstash/outputs/s3.rb
+- lib/logstash/outputs/s3/file_repository.rb
+- lib/logstash/outputs/s3/patch.rb
+- lib/logstash/outputs/s3/path_validator.rb
+- lib/logstash/outputs/s3/size_and_time_rotation_policy.rb
+- lib/logstash/outputs/s3/size_rotation_policy.rb
+- lib/logstash/outputs/s3/temporary_file.rb
+- lib/logstash/outputs/s3/temporary_file_factory.rb
+- lib/logstash/outputs/s3/time_rotation_policy.rb
+- lib/logstash/outputs/s3/uploader.rb
+- lib/logstash/outputs/s3/writable_directory_validator.rb
+- lib/logstash/outputs/s3/write_bucket_permission_validator.rb
 - logstash-output-s3.gemspec
-- spec/integration/s3_spec.rb
+- spec/integration/dynamic_prefix_spec.rb
+- spec/integration/gzip_file_spec.rb
+- spec/integration/gzip_size_rotation_spec.rb
+- spec/integration/restore_from_crash_spec.rb
+- spec/integration/size_rotation_spec.rb
+- spec/integration/stress_test_spec.rb
+- spec/integration/time_based_rotation_with_constant_write_spec.rb
+- spec/integration/time_based_rotation_with_stale_write_spec.rb
+- spec/integration/upload_current_file_on_shutdown_spec.rb
+- spec/outputs/s3/file_repository_spec.rb
+- spec/outputs/s3/size_and_time_rotation_policy_spec.rb
+- spec/outputs/s3/size_rotation_policy_spec.rb
+- spec/outputs/s3/temporary_file_factory_spec.rb
+- spec/outputs/s3/temporary_file_spec.rb
+- spec/outputs/s3/time_rotation_policy_spec.rb
+- spec/outputs/s3/uploader_spec.rb
+- spec/outputs/s3/writable_directory_validator_spec.rb
+- spec/outputs/s3/write_bucket_permission_validator_spec.rb
 - spec/outputs/s3_spec.rb
+- spec/spec_helper.rb
 - spec/supports/helpers.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
@@ -145,6 +188,24 @@ signing_key:
 specification_version: 4
 summary: This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)
 test_files:
-- spec/integration/s3_spec.rb
+- spec/integration/dynamic_prefix_spec.rb
+- spec/integration/gzip_file_spec.rb
+- spec/integration/gzip_size_rotation_spec.rb
+- spec/integration/restore_from_crash_spec.rb
+- spec/integration/size_rotation_spec.rb
+- spec/integration/stress_test_spec.rb
+- spec/integration/time_based_rotation_with_constant_write_spec.rb
+- spec/integration/time_based_rotation_with_stale_write_spec.rb
+- spec/integration/upload_current_file_on_shutdown_spec.rb
+- spec/outputs/s3/file_repository_spec.rb
+- spec/outputs/s3/size_and_time_rotation_policy_spec.rb
+- spec/outputs/s3/size_rotation_policy_spec.rb
+- spec/outputs/s3/temporary_file_factory_spec.rb
+- spec/outputs/s3/temporary_file_spec.rb
+- spec/outputs/s3/time_rotation_policy_spec.rb
+- spec/outputs/s3/uploader_spec.rb
+- spec/outputs/s3/writable_directory_validator_spec.rb
+- spec/outputs/s3/write_bucket_permission_validator_spec.rb
 - spec/outputs/s3_spec.rb
+- spec/spec_helper.rb
 - spec/supports/helpers.rb

data/spec/integration/s3_spec.rb

@@ -1,97 +0,0 @@
-# encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/outputs/s3"
-require 'socket'
-require "aws-sdk"
-require "fileutils"
-require "stud/temporary"
-require_relative "../supports/helpers"
-
-describe LogStash::Outputs::S3, :integration => true, :s3 => true do
-  before do
-    Thread.abort_on_exception = true
-  end
-
-  let!(:minimal_settings)  {  { "access_key_id" => ENV['AWS_ACCESS_KEY_ID'],
-                                "secret_access_key" => ENV['AWS_SECRET_ACCESS_KEY'],
-                                "bucket" => ENV['AWS_LOGSTASH_TEST_BUCKET'],
-                                "region" => ENV["AWS_REGION"] || "us-east-1",
-                                "temporary_directory" => Stud::Temporary.pathname('temporary_directory') }}
-
-  let!(:s3_object) do
-      s3output = LogStash::Outputs::S3.new(minimal_settings)
-      s3output.register
-      s3output.s3
-  end
-
-  after(:each) do
-    delete_matching_keys_on_bucket('studtmp')
-    delete_matching_keys_on_bucket('my-prefix')
-  end
-
-  describe "#register" do
-    it "write a file on the bucket to check permissions" do
-      s3 = LogStash::Outputs::S3.new(minimal_settings)
-      expect(s3.register).not_to raise_error
-    end
-  end
-
-  describe "#write_on_bucket" do
-    after(:each) do
-      File.unlink(fake_data.path)
-    end
-
-    let!(:fake_data) { Stud::Temporary.file }
-
-    it "should prefix the file on the bucket if a prefix is specified" do
-      prefix = "my-prefix"
-
-      config = minimal_settings.merge({
-        "prefix" => prefix,
-      })
-
-      s3 = LogStash::Outputs::S3.new(config)
-      s3.register
-      s3.write_on_bucket(fake_data)
-
-      expect(key_exists_on_bucket?("#{prefix}#{File.basename(fake_data.path)}")).to eq(true)
-    end
-
-    it 'should use the same local filename if no prefix is specified' do
-      s3 = LogStash::Outputs::S3.new(minimal_settings)
-      s3.register
-      s3.write_on_bucket(fake_data)
-
-      expect(key_exists_on_bucket?(File.basename(fake_data.path))).to eq(true)
-    end
-  end
-
-  describe "#move_file_to_bucket" do
-    let!(:s3) { LogStash::Outputs::S3.new(minimal_settings) }
-
-    before do
-      s3.register
-    end
-
-    it "should upload the file if the size > 0" do
-      tmp = Stud::Temporary.file
-      allow(File).to receive(:zero?).and_return(false)
-      s3.move_file_to_bucket(tmp)
-      expect(key_exists_on_bucket?(File.basename(tmp.path))).to eq(true)
-    end
-  end
-
-  describe "#restore_from_crashes" do
-    it "read the temp directory and upload the matching file to s3" do
-      Stud::Temporary.pathname do |temp_path|
-        tempfile = File.open(File.join(temp_path, 'A'), 'w+') { |f| f.write('test')}
-
-        s3 = LogStash::Outputs::S3.new(minimal_settings.merge({ "temporary_directory" => temp_path }))
-        s3.restore_from_crashes
-
-        expect(File.exist?(tempfile.path)).to eq(false)
-        expect(key_exists_on_bucket?(File.basename(tempfile.path))).to eq(true)
-      end
-    end
-  end
-end