logstash-output-s3 3.2.0 → 4.0.0

data/lib/logstash/outputs/s3/writable_directory_validator.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+module LogStash
+  module Outputs
+    class S3
+      class WritableDirectoryValidator
+        def self.valid?(path)
+          begin
+            FileUtils.mkdir_p(path) unless Dir.exist?(path)
+            ::File.writable?(path)
+          rescue
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/s3/write_bucket_permission_validator.rb

@@ -0,0 +1,49 @@
+# encoding: utf-8
+require "stud/temporary"
+require "socket"
+require "fileutils"
+
+module LogStash
+  module Outputs
+    class S3
+      class WriteBucketPermissionValidator
+        def self.valid?(bucket_resource)
+          begin
+            upload_test_file(bucket_resource)
+            true
+          rescue
+            false
+          end
+        end
+
+        private
+        def self.upload_test_file(bucket_resource)
+          generated_at = Time.now
+
+          key = "logstash-programmatic-access-test-object-#{generated_at}"
+          content = "Logstash permission check on #{generated_at}, by #{Socket.gethostname}"
+
+          begin
+            f = Stud::Temporary.file
+            f.write(content)
+            f.fsync
+            f.close
+
+            obj = bucket_resource.object(key)
+            obj.upload_file(f)
+
+            begin
+              obj.delete
+            rescue
+              # Try to remove the files on the remote bucket,
+              # but don't raise any errors if that doesn't work.
+              # since we only really need `putobject`.
+            end
+          ensure
+            FileUtils.rm_rf(f.path)
+          end
+        end
+      end
+    end
+  end
+end

data/logstash-output-s3.gemspec

@@ -1,7 +1,6 @@
 Gem::Specification.new do |s|
-
   s.name            = 'logstash-output-s3'
-  s.version         = '3.2.0'
+  s.version         = '4.0.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -22,6 +21,7 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-aws'
+  s.add_runtime_dependency "concurrent-ruby"
   s.add_runtime_dependency 'stud', '~> 0.0.22'
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'logstash-input-generator'

data/spec/integration/dynamic_prefix_spec.rb

@@ -0,0 +1,92 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/s3"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "Dynamic Prefix", :integration => true do
+  include_context "setup plugin"
+
+  let(:options) { main_options.merge({ "rotation_strategy" => "size" }) }
+  let(:sandbox) { "test" }
+
+  before do
+    clean_remote_files(sandbox)
+    subject.register
+    subject.multi_receive_encoded(batch)
+    subject.close
+  end
+
+  context "With field string" do
+    let(:prefix) { "/#{sandbox}/%{server}/%{language}" }
+    let(:batch) do
+      b = {}
+      e1 = LogStash::Event.new({ "server" => "es1", "language" => "ruby"})
+      b[e1] = "es1-ruby"
+      e2 = LogStash::Event.new({ "server" => "es2", "language" => "java"})
+      b[e2] = "es2-ruby"
+      b
+    end
+
+    it "creates a specific quantity of files" do
+      expect(bucket_resource.objects(:prefix => sandbox).count).to eq(batch.size)
+    end
+
+    it "creates specific keys" do
+      re = Regexp.union(/^es1\/ruby\/ls.s3.sashimi/, /^es2\/java\/ls.s3.sashimi/)
+
+      bucket_resource.objects(:prefix => sandbox) do |obj|
+        expect(obj.key).to match(re)
+      end
+    end
+
+    it "Persists all events" do
+      download_directory = Stud::Temporary.pathname
+
+      FileUtils.rm_rf(download_directory)
+      FileUtils.mkdir_p(download_directory)
+
+      counter = 0
+      bucket_resource.objects(:prefix => sandbox).each do |object|
+        target = File.join(download_directory, "#{counter}.txt")
+        object.get(:response_target => target)
+        counter += 1
+      end
+      expect(Dir.glob(File.join(download_directory, "**", "*.txt")).inject(0) { |sum, f| sum + IO.readlines(f).size }).to eq(batch.size)
+    end
+  end
+
+  context "with unsupported char" do
+    let(:prefix) { "/#{sandbox}/%{server}/%{language}" }
+    let(:batch) do
+      b = {}
+      e1 = LogStash::Event.new({ "server" => "e>s1", "language" => "ruby"})
+      b[e1] = "es2-ruby"
+      b
+    end
+
+    it "convert them to underscore" do
+      re = Regexp.union(/^e_s1\/ruby\/ls.s3.sashimi/)
+
+      bucket_resource.objects(:prefix => sandbox) do |obj|
+        expect(obj.key).to match(re)
+      end
+    end
+  end
+
+  context "with dates" do
+    let(:prefix) { "/#{sandbox}/%{+YYYY-MM-d}" }
+
+    let(:batch) do
+      b = {}
+      e1 = LogStash::Event.new({ "server" => "e>s1", "language" => "ruby"})
+      b[e1] = "es2-ruby"
+      b
+    end
+
+    it "creates dated path" do
+      re = /^#{sandbox}\/\d{4}-\d{2}-\d{1,2}\/ls\.s3\./
+      expect(bucket_resource.objects(:prefix => sandbox).first.key).to match(re)
+    end
+  end
+end

data/spec/integration/gzip_file_spec.rb

@@ -0,0 +1,62 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/s3"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "Gzip File Time rotation with constant write", :integration => true do
+  include_context "setup plugin"
+
+  let(:time_file) { 0.004 }
+  let(:options) { main_options.merge({ "encoding" => "gzip",
+                                       "rotation_strategy" => "time" }) }
+  let(:number_of_events) { 5000 }
+  let(:batch_size) { 125 }
+  let(:event_encoded) { "Hello world" }
+  let(:batch) do
+    b = {}
+    number_of_events.times do
+      event = LogStash::Event.new({ "message" => event_encoded })
+      b[event] = "#{event_encoded}\n"
+    end
+    b
+  end
+  let(:minimum_number_of_time_rotation) { 3 }
+  let(:batch_step) { (number_of_events / minimum_number_of_time_rotation).ceil }
+
+  before do
+    clean_remote_files(prefix)
+    subject.register
+
+    # simulate batch read/write
+    batch.each_slice(batch_step) do |batch_time|
+      batch_time.each_slice(batch_size) do |smaller_batch|
+        subject.multi_receive_encoded(smaller_batch)
+      end
+      sleep(1)
+    end
+
+    subject.close
+  end
+
+  it "creates multiples files" do
+    # using close will upload the current file
+    expect(bucket_resource.objects(:prefix => prefix).count).to be_between(minimum_number_of_time_rotation, minimum_number_of_time_rotation + 1).inclusive
+  end
+
+  it "Persists all events" do
+    download_directory = Stud::Temporary.pathname
+
+    FileUtils.rm_rf(download_directory)
+    FileUtils.mkdir_p(download_directory)
+
+    counter = 0
+    bucket_resource.objects(:prefix => prefix).each do |object|
+      target = File.join(download_directory, "#{counter}.gz")
+      object.get(:response_target => target)
+      counter += 1
+    end
+
+    expect(Dir.glob(File.join(download_directory, "**", "*.gz")).inject(0) { |sum, f| sum + Zlib::GzipReader.new(File.open(f)).readlines.size }).to eq(number_of_events)
+  end
+end

data/spec/integration/gzip_size_rotation_spec.rb

@@ -0,0 +1,63 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/s3"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "Gzip Size rotation", :integration => true do
+  include_context "setup plugin"
+
+  let(:document_size) { 20 * 1024 } # in bytes
+
+  let(:options) do
+    main_options.merge({
+      "encoding" => "gzip",
+      "size_file" => document_size,
+      "rotation_strategy" => "size" })
+  end
+
+  let(:number_of_events) { 1_000_000 }
+  let(:batch_size) { 125 }
+  let(:event_encoded) { "Hello world" * 20 }
+  let(:batch) do
+    b = {}
+    batch_size.times do
+      event = LogStash::Event.new({ "message" => event_encoded })
+      b[event] = "#{event_encoded}\n"
+    end
+    b
+  end
+  let(:number_of_files) { number_of_events / 50000 }
+
+  before do
+    clean_remote_files(prefix)
+    subject.register
+    (number_of_events/batch_size).times do
+      subject.multi_receive_encoded(batch)
+    end
+    subject.close
+  end
+
+  it "Rotates the files based on size" do
+    f = bucket_resource.objects(:prefix => prefix).first
+    expect(f.size).to be_between(document_size, document_size * 2).inclusive
+  end
+
+  it "Persists all events" do
+    download_directory = Stud::Temporary.pathname
+
+    FileUtils.rm_rf(download_directory)
+    FileUtils.mkdir_p(download_directory)
+
+    counter = 0
+    bucket_resource.objects(:prefix => prefix).each do |object|
+      target = File.join(download_directory, "#{counter}.txt.gz")
+      object.get(:response_target => target)
+      counter += 1
+    end
+
+    expect(Dir.glob(File.join(download_directory, "**", "*.gz")).inject(0) do |sum, f|
+      sum + Zlib::GzipReader.new(File.open(f)).readlines.size
+    end).to eq(number_of_events)
+  end
+end

data/spec/integration/restore_from_crash_spec.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/s3"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "Restore from crash", :integration => true do
+  include_context "setup plugin"
+
+  let(:options) { main_options.merge({ "restore" => true }) }
+
+  let(:number_of_files) { 5 }
+  let(:dummy_content) { "foobar\n" * 100 }
+
+  before do
+    clean_remote_files(prefix)
+    # Use the S3 factory to create mutliples files with dummy content
+    factory = LogStash::Outputs::S3::TemporaryFileFactory.new(prefix, tags, "none", temporary_directory)
+
+    # Creating a factory always create a file
+    factory.current.write(dummy_content)
+    factory.current.fsync
+
+    (number_of_files - 1).times do
+      factory.rotate!
+      factory.current.write(dummy_content)
+      factory.current.fsync
+    end
+  end
+
+  it "uploads the file to the bucket" do
+    subject.register
+    try(20) do
+      expect(bucket_resource.objects(:prefix => prefix).count).to eq(number_of_files)
+      expect(Dir.glob(File.join(temporary_directory, "*")).size).to eq(0)
+    end
+  end
+end
+

data/spec/integration/size_rotation_spec.rb

@@ -0,0 +1,59 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/s3"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "Size rotation", :integration => true do
+  include_context "setup plugin"
+
+  let(:event_size) { "Hello world".bytesize }
+  let(:size_file) { batch_size * event_size * 2 }
+  let(:options) { main_options.merge({ "rotation_strategy" => "size", "size_file" => size_file }) }
+  let(:number_of_events) { 5000 }
+  let(:batch_size) { 125 }
+  let(:event_encoded) { "Hello world" }
+  let(:batch) do
+    b = {}
+    number_of_events.times do
+      event = LogStash::Event.new({ "message" => event_encoded })
+      b[event] = "#{event_encoded}\n"
+    end
+    b
+  end
+  let(:number_of_files) { number_of_events * event_size / size_file }
+
+  before do
+    clean_remote_files(prefix)
+    subject.register
+    batch.each_slice(batch_size) do |smaller_batch|
+      subject.multi_receive_encoded(smaller_batch)
+    end
+    subject.close
+  end
+
+  it "creates a specific quantity of files" do
+    expect(bucket_resource.objects(:prefix => prefix).count).to eq(number_of_files)
+  end
+
+  it "Rotates the files based on size" do
+    bucket_resource.objects(:prefix => prefix).each do |f|
+      expect(f.size).to be_between(size_file, size_file * 2).inclusive
+    end
+  end
+
+  it "Persists all events" do
+    download_directory = Stud::Temporary.pathname
+
+    FileUtils.rm_rf(download_directory)
+    FileUtils.mkdir_p(download_directory)
+
+    counter = 0
+    bucket_resource.objects(:prefix => prefix).each do |object|
+      target = File.join(download_directory, "#{counter}.txt")
+      object.get(:response_target => target)
+      counter += 1
+    end
+    expect(Dir.glob(File.join(download_directory, "**", "*.txt")).inject(0) { |sum, f| sum + IO.readlines(f).size }).to eq(number_of_events)
+  end
+end

data/spec/integration/stress_test_spec.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/s3"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "Upload current file on shutdown", :integration => true, :slow => true do
+  include_context "setup plugin"
+  let(:stress_time) { ENV["RUNTIME"] || 1 * 60 }
+  let(:options) { main_options }
+
+  let(:time_file) { 15 }
+  let(:batch_size) { 125 }
+  let(:event_encoded) { "Hello world" }
+  let(:batch) do
+    b = {}
+    batch_size.times do
+      event = LogStash::Event.new({ "message" => event_encoded })
+      b[event] = "#{event_encoded}\n"
+    end
+    b
+  end
+  let(:workers) { 3 }
+
+  it "Persists all events" do
+    started_at = Time.now
+    events_sent = {}
+
+    clean_remote_files(prefix)
+    subject.register
+
+    workers.times do
+      Thread.new do
+        events_sent[Thread.current] = 0
+
+        while Time.now - started_at < stress_time
+          subject.multi_receive_encoded(batch)
+          events_sent[Thread.current] += batch_size
+        end
+      end
+    end
+
+    sleep(1) while Time.now - started_at < stress_time
+
+    subject.close
+
+    download_directory = Stud::Temporary.pathname
+
+    FileUtils.rm_rf(download_directory)
+    FileUtils.mkdir_p(download_directory)
+
+    counter = 0
+    bucket_resource.objects(:prefix => prefix).each do |object|
+      target = File.join(download_directory, "#{counter}.txt")
+      object.get(:response_target => target)
+      counter += 1
+    end
+    expect(Dir.glob(File.join(download_directory, "**", "*.txt")).inject(0) { |sum, f| sum + IO.readlines(f).size }).to eq(events_sent.values.inject(0, :+))
+  end
+end