logstash-output-s3 3.2.0 → 4.0.0

data/spec/integration/time_based_rotation_with_constant_write_spec.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/s3"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "File Time rotation with constant write", :integration => true do
+  include_context "setup plugin"
+
+  let(:time_file) { 0.004 }
+  let(:options) { main_options.merge({ "rotation_strategy" => "time" }) }
+  let(:number_of_events) { 5000 }
+  let(:batch_size) { 125 }
+  let(:event_encoded) { "Hello world" }
+  let(:batch) do
+    b = {}
+    number_of_events.times do
+      event = LogStash::Event.new({ "message" => event_encoded })
+      b[event] = "#{event_encoded}\n"
+    end
+    b
+  end
+  let(:minimum_number_of_time_rotation) { 3 }
+  let(:batch_step) { (number_of_events / minimum_number_of_time_rotation).ceil }
+
+  before do
+    clean_remote_files(prefix)
+    subject.register
+
+    # simulate batch read/write
+    batch.each_slice(batch_step) do |batch_time|
+      batch_time.each_slice(batch_size) do |smaller_batch|
+        subject.multi_receive_encoded(smaller_batch)
+      end
+      sleep(1)
+    end
+
+    subject.close
+  end
+
+  it "creates multiples files" do
+    # using close will upload the current file
+    expect(bucket_resource.objects(:prefix => prefix).count).to be_between(minimum_number_of_time_rotation, minimum_number_of_time_rotation + 1).inclusive
+  end
+
+  it "Persists all events" do
+    download_directory = Stud::Temporary.pathname
+
+    FileUtils.rm_rf(download_directory)
+    FileUtils.mkdir_p(download_directory)
+
+    counter = 0
+    bucket_resource.objects(:prefix => prefix).each do |object|
+      target = File.join(download_directory, "#{counter}.txt")
+      object.get(:response_target => target)
+      counter += 1
+    end
+    expect(Dir.glob(File.join(download_directory, "**", "*.txt")).inject(0) { |sum, f| sum + IO.readlines(f).size }).to eq(number_of_events)
+  end
+end

data/spec/integration/time_based_rotation_with_stale_write_spec.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/s3"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "File Time rotation with stale write", :integration => true do
+  include_context "setup plugin"
+
+  let(:time_file) { 0.0004 }
+  let(:options) { main_options.merge({ "rotation_strategy" => "time" }) }
+  let(:number_of_events) { 5000 }
+  let(:batch_size) { 125 }
+  let(:event_encoded) { "Hello world" }
+  let(:batch) do
+    b = {}
+    number_of_events.times do
+      event = LogStash::Event.new({ "message" => event_encoded })
+      b[event] = "#{event_encoded}\n"
+    end
+    b
+  end
+
+  before do
+    stub_const('LogStash::Outputs::S3::PERIODIC_CHECK_INTERVAL_IN_SECONDS', 1)
+    clean_remote_files(prefix)
+    subject.register
+    subject.multi_receive_encoded(batch)
+    sleep(1) # the periodic check should have kick int
+  end
+
+  after do
+    subject.close
+  end
+
+  it "create one file" do
+    # using close will upload the current file
+    try(20) do
+      expect(bucket_resource.objects(:prefix => prefix).count).to eq(1)
+    end
+  end
+
+  it "Persists all events" do
+    download_directory = Stud::Temporary.pathname
+
+    FileUtils.rm_rf(download_directory)
+    FileUtils.mkdir_p(download_directory)
+
+    counter = 0
+    bucket_resource.objects(:prefix => prefix).each do |object|
+      target = File.join(download_directory, "#{counter}.txt")
+      object.get(:response_target => target)
+      counter += 1
+    end
+
+    try(20) do
+      expect(Dir.glob(File.join(download_directory, "**", "*.txt")).inject(0) { |sum, f| sum + IO.readlines(f).size }).to eq(number_of_events)
+    end
+  end
+end

data/spec/integration/upload_current_file_on_shutdown_spec.rb

@@ -0,0 +1,51 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/s3"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "Upload current file on shutdown", :integration => true do
+  include_context "setup plugin"
+  let(:options) { main_options }
+
+  let(:size_file) { 1000000 }
+  let(:time_file) { 100000 }
+  let(:number_of_events) { 5000 }
+  let(:batch_size) { 125 }
+  let(:event_encoded) { "Hello world" }
+  let(:batch) do
+    b = {}
+    number_of_events.times do
+      event = LogStash::Event.new({ "message" => event_encoded })
+      b[event] = "#{event_encoded}\n"
+    end
+    b
+  end
+
+  before do
+    clean_remote_files(prefix)
+    subject.register
+    subject.multi_receive_encoded(batch)
+    subject.close
+  end
+
+  it "creates a specific quantity of files" do
+    # Since we have really big value of time_file and size_file
+    expect(bucket_resource.objects(:prefix => prefix).count).to eq(1)
+  end
+
+  it "Persists all events" do
+    download_directory = Stud::Temporary.pathname
+
+    FileUtils.rm_rf(download_directory)
+    FileUtils.mkdir_p(download_directory)
+
+    counter = 0
+    bucket_resource.objects(:prefix => prefix).each do |object|
+      target = File.join(download_directory, "#{counter}.txt")
+      object.get(:response_target => target)
+      counter += 1
+    end
+    expect(Dir.glob(File.join(download_directory, "**", "*.txt")).inject(0) { |sum, f| sum + IO.readlines(f).size }).to eq(number_of_events)
+  end
+end

data/spec/outputs/s3/file_repository_spec.rb

@@ -0,0 +1,146 @@
+# encoding: utf-8
+require "logstash/outputs/s3"
+require "stud/temporary"
+require "fileutils"
+require_relative "../../spec_helper"
+
+describe LogStash::Outputs::S3::FileRepository do
+  let(:tags) { ["secret", "service"] }
+  let(:encoding) { "none" }
+  let(:temporary_directory) { Stud::Temporary.pathname }
+  let(:prefix_key) { "a-key" }
+
+  before do
+    FileUtils.mkdir_p(temporary_directory)
+  end
+
+  subject { described_class.new(tags, encoding, temporary_directory) }
+
+  it "returns a temporary file" do
+    subject.get_file(prefix_key) do |file|
+      expect(file).to be_kind_of(LogStash::Outputs::S3::TemporaryFile)
+    end
+  end
+
+  it "returns the same file for the same prefix key" do
+    file_path = nil
+
+    subject.get_file(prefix_key) do |file|
+      file_path = file.path
+    end
+
+    subject.get_file(prefix_key) do |file|
+      expect(file.path).to eq(file_path)
+    end
+  end
+
+  it "returns the same file for the same dynamic prefix key" do
+    prefix = "%{type}/%{+YYYY}/%{+MM}/%{+dd}/"
+    event = LogStash::Event.new({ "type" => "syslog"})
+    key = event.sprintf(prefix)
+    file_path = nil
+
+
+    subject.get_file(key) do |file|
+      file_path = file.path
+    end
+
+    subject.get_file(key) do |file|
+      expect(file.path).to eq(file_path)
+    end
+  end
+
+  it "returns different file for different prefix keys" do
+    file_path = nil
+
+    subject.get_file(prefix_key) do |file|
+      file_path = file.path
+    end
+
+    subject.get_file("another_prefix_key") do |file|
+      expect(file.path).not_to eq(file_path)
+    end
+  end
+
+  it "allows to get the file factory for a specific prefix" do
+    subject.get_factory(prefix_key) do |factory|
+      expect(factory).to be_kind_of(LogStash::Outputs::S3::TemporaryFileFactory)
+    end
+  end
+
+  it "returns a different file factory for a different prefix keys" do
+    factory = nil
+
+    subject.get_factory(prefix_key) do |f|
+      factory = f
+    end
+
+    subject.get_factory("another_prefix_key") do |f|
+      expect(factory).not_to eq(f)
+    end
+  end
+
+  it "returns the number of prefix keys" do
+    expect(subject.size).to eq(0)
+    subject.get_file(prefix_key) { |file| file.write("something") }
+    expect(subject.size).to eq(1)
+  end
+
+  it "returns all available keys" do
+    subject.get_file(prefix_key) { |file| file.write("something") }
+    expect(subject.keys.toArray).to include(prefix_key)
+    expect(subject.keys.toArray.size).to eq(1)
+  end
+
+  it "clean stale factories" do
+    @file_repository = described_class.new(tags, encoding, temporary_directory, 1, 1)
+    expect(@file_repository.size).to eq(0)
+    path = ""
+    @file_repository.get_factory(prefix_key) do |factory|
+      factory.current.write("hello")
+      # force a rotation so we get an empty file that will get stale.
+      factory.rotate!
+      path = factory.current.temp_path
+    end
+
+    @file_repository.get_file("another-prefix") { |file| file.write("hello") }
+    expect(@file_repository.size).to eq(2)
+    @file_repository.keys.each do |k|
+      puts k
+    end
+    try(10) { expect(@file_repository.size).to eq(1) }
+    expect(File.directory?(path)).to be_falsey
+  end
+end
+
+
+describe LogStash::Outputs::S3::FileRepository::PrefixedValue do
+  let(:factory) { spy("factory", :current => file) }
+  subject { described_class.new(factory, 1) }
+
+  context "#stale?" do
+    context "the file is empty and older than stale time" do
+      let(:file) { double("file", :size => 0, :ctime => Time.now - 5) }
+
+      it "returns true" do
+        expect(subject.stale?).to be_truthy
+      end
+    end
+
+    context "when the file has data in it" do
+      let(:file) { double("file", :size => 200, :ctime => Time.now - 5) }
+
+      it "returns false" do
+        expect(subject.stale?).to be_falsey
+      end
+    end
+
+    context "when the file is not old enough" do
+      let(:file) { double("file", :size => 0, :ctime => Time.now + 100) }
+
+      it "returns false" do
+        expect(subject.stale?).to be_falsey
+      end
+    end
+  end
+end

data/spec/outputs/s3/size_and_time_rotation_policy_spec.rb

@@ -0,0 +1,77 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/s3/size_and_time_rotation_policy"
+require "logstash/outputs/s3/temporary_file"
+
+describe LogStash::Outputs::S3::SizeAndTimeRotationPolicy do
+  let(:file_size) { 10 }
+  let(:time_file) { 1 }
+  subject { described_class.new(file_size, time_file) }
+
+  let(:temporary_directory) { Stud::Temporary.pathname }
+  let(:temporary_file) { Stud::Temporary.file }
+  let(:name) { "foobar" }
+  let(:content) { "hello" * 1000 }
+  let(:file) { LogStash::Outputs::S3::TemporaryFile.new(name, temporary_file, temporary_directory) }
+
+  it "raises an exception if the `time_file` is set to 0" do
+    expect { described_class.new(100, 0) }.to raise_error(LogStash::ConfigurationError, /time_file/)
+  end
+
+  it "raises an exception if the `time_file` is < 0" do
+    expect { described_class.new(100, -100) }.to raise_error(LogStash::ConfigurationError, /time_file/)
+  end
+
+  it "raises an exception if the `size_file` is 0" do
+    expect { described_class.new(0, 100) }.to raise_error(LogStash::ConfigurationError, /size_file/)
+  end
+
+  it "raises an exception if the `size_file` is < 0" do
+    expect { described_class.new(-100, 100) }.to raise_error(LogStash::ConfigurationError, /size_file/)
+  end
+
+  it "returns true if the size on disk is higher than the `file_size`" do
+    file.write(content)
+    file.fsync
+    expect(subject.rotate?(file)).to be_truthy
+  end
+
+  it "returns false if the size is inferior than the `file_size`" do
+    expect(subject.rotate?(file)).to be_falsey
+  end
+
+  context "when the size of the file is superior to 0" do
+    let(:file_size) { 10000 }
+
+    before :each do
+      file.write(content)
+      file.fsync
+    end
+
+    it "returns true if the file old enough" do
+      allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60) )
+      expect(subject.rotate?(file)).to be_truthy
+    end
+
+    it "returns false is not old enough" do
+      allow(file).to receive(:ctime).and_return(Time.now + time_file * 10)
+      expect(subject.rotate?(file)).to be_falsey
+    end
+  end
+
+  context "When the size of the file is 0" do
+    it "returns false if the file old enough" do
+      expect(subject.rotate?(file)).to be_falsey
+    end
+
+    it "returns false is not old enough" do
+      expect(subject.rotate?(file)).to be_falsey
+    end
+  end
+
+  context "#needs_periodic?" do
+    it "return true" do
+      expect(subject.needs_periodic?).to be_truthy
+    end
+  end
+end

data/spec/outputs/s3/size_rotation_policy_spec.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/s3/size_rotation_policy"
+require "logstash/outputs/s3/temporary_file"
+require "fileutils"
+
+describe LogStash::Outputs::S3::SizeRotationPolicy do
+  subject { described_class.new(size_file) }
+
+  let(:temporary_directory) {  Stud::Temporary.directory }
+  let(:temporary_file) { Stud::Temporary.file }
+  let(:name) { "foobar" }
+  let(:content) { "hello" * 1000 }
+  let(:size_file) { 10 } # in bytes
+  let(:file) { LogStash::Outputs::S3::TemporaryFile.new(name, temporary_file, temporary_directory) }
+
+  it "returns true if the size on disk is higher than the `size_file`" do
+    file.write(content)
+    file.fsync
+    expect(subject.rotate?(file)).to be_truthy
+  end
+
+  it "returns false if the size is inferior than the `size_file`" do
+    expect(subject.rotate?(file)).to be_falsey
+  end
+
+  it "raises an exception if the `size_file` is 0" do
+    expect { described_class.new(0) }.to raise_error(LogStash::ConfigurationError, /need to be greather than 0/)
+  end
+
+  it "raises an exception if the `size_file` is < 0" do
+    expect { described_class.new(-100) }.to raise_error(LogStash::ConfigurationError, /need to be greather than 0/)
+  end
+
+  context "#needs_periodic?" do
+    it "return false" do
+      expect(subject.needs_periodic?).to be_falsey
+    end
+  end
+
+end

data/spec/outputs/s3/temporary_file_factory_spec.rb

@@ -0,0 +1,85 @@
+# encoding: utf-8
+require "logstash/outputs/s3/temporary_file_factory"
+require "logstash/outputs/s3/temporary_file"
+require "stud/temporary"
+require "fileutils"
+
+describe LogStash::Outputs::S3::TemporaryFileFactory do
+  let(:prefix) { "foobar" }
+  let(:tags) { [] }
+  let(:temporary_directory) { Stud::Temporary.pathname }
+
+  before do
+    FileUtils.mkdir_p(temporary_directory)
+  end
+
+  subject { described_class.new(prefix, tags, encoding, temporary_directory) }
+
+  shared_examples "file factory" do
+    it "creates the file on disk" do
+      expect(File.exist?(subject.current.path)).to be_truthy
+    end
+
+    it "create a temporary file when initialized" do
+      expect(subject.current).to be_kind_of(LogStash::Outputs::S3::TemporaryFile)
+    end
+
+    it "create a file in the right format" do
+      expect(subject.current.path).to match(extension)
+    end
+
+    it "allow to rotate the file" do
+      file_path = subject.current.path
+      expect(subject.rotate!.path).not_to eq(file_path)
+    end
+
+    it "increments the part name on rotation" do
+      expect(subject.current.path).to match(/part0/)
+      expect(subject.rotate!.path).to match(/part1/)
+    end
+
+    it "includes the date" do
+      n = Time.now
+      expect(subject.current.path).to include(n.strftime("%Y-%m-%dT"))
+    end
+
+    it "include the file key in the path"  do
+      file = subject.current
+      expect(file.path).to match(/#{file.key}/)
+    end
+
+    it "create a unique directory in the temporary directory for each file" do
+      uuid = "hola"
+      expect(SecureRandom).to receive(:uuid).and_return(uuid).twice
+      expect(subject.current.path).to include(uuid)
+    end
+
+    context "with tags supplied" do
+      let(:tags) { ["secret", "service"] }
+
+      it "adds tags to the filename" do
+        expect(subject.current.path).to match(/tag_#{tags.join('.')}.part/)
+      end
+    end
+
+    context "without tags" do
+      it "doesn't add tags to the filename" do
+        expect(subject.current.path).not_to match(/tag_/)
+      end
+    end
+  end
+
+  context "when gzip" do
+    let(:encoding) { "gzip" }
+    let(:extension) { /\.txt.gz$/ }
+
+    include_examples "file factory"
+  end
+
+  context "when encoding set to `none`" do
+    let(:encoding) { "none" }
+    let(:extension) { /\.txt$/ }
+
+    include_examples "file factory"
+  end
+end