logstash-integration-aws 0.1.0.pre

data/spec/outputs/s3/file_repository_spec.rb

@@ -0,0 +1,143 @@
+# encoding: utf-8
+require "logstash/outputs/s3"
+require "stud/temporary"
+require "fileutils"
+require_relative "../../spec_helper"
+
+describe LogStash::Outputs::S3::FileRepository do
+  let(:tags) { ["secret", "service"] }
+  let(:encoding) { "none" }
+  let(:temporary_directory) { Stud::Temporary.pathname }
+  let(:prefix_key) { "a-key" }
+
+  before do
+    FileUtils.mkdir_p(temporary_directory)
+  end
+
+  subject { described_class.new(tags, encoding, temporary_directory) }
+
+  it "returns a temporary file" do
+    subject.get_file(prefix_key) do |file|
+      expect(file).to be_kind_of(LogStash::Outputs::S3::TemporaryFile)
+    end
+  end
+
+  it "returns the same file for the same prefix key" do
+    file_path = nil
+
+    subject.get_file(prefix_key) do |file|
+      file_path = file.path
+    end
+
+    subject.get_file(prefix_key) do |file|
+      expect(file.path).to eq(file_path)
+    end
+  end
+
+  it "returns the same file for the same dynamic prefix key" do
+    prefix = "%{type}/%{+YYYY}/%{+MM}/%{+dd}/"
+    event = LogStash::Event.new({ "type" => "syslog"})
+    key = event.sprintf(prefix)
+    file_path = nil
+
+
+    subject.get_file(key) do |file|
+      file_path = file.path
+    end
+
+    subject.get_file(key) do |file|
+      expect(file.path).to eq(file_path)
+    end
+  end
+
+  it "returns different file for different prefix keys" do
+    file_path = nil
+
+    subject.get_file(prefix_key) do |file|
+      file_path = file.path
+    end
+
+    subject.get_file("another_prefix_key") do |file|
+      expect(file.path).not_to eq(file_path)
+    end
+  end
+
+  it "allows to get the file factory for a specific prefix" do
+    subject.get_factory(prefix_key) do |factory|
+      expect(factory).to be_kind_of(LogStash::Outputs::S3::TemporaryFileFactory)
+    end
+  end
+
+  it "returns a different file factory for a different prefix keys" do
+    factory = nil
+
+    subject.get_factory(prefix_key) do |f|
+      factory = f
+    end
+
+    subject.get_factory("another_prefix_key") do |f|
+      expect(factory).not_to eq(f)
+    end
+  end
+
+  it "returns the number of prefix keys" do
+    expect(subject.size).to eq(0)
+    subject.get_file(prefix_key) { |file| file.write("something") }
+    expect(subject.size).to eq(1)
+  end
+
+  it "returns all available keys" do
+    subject.get_file(prefix_key) { |file| file.write("something") }
+    expect(subject.keys.toArray).to include(prefix_key)
+    expect(subject.keys.toArray.size).to eq(1)
+  end
+
+  it "clean stale factories" do
+    @file_repository = described_class.new(tags, encoding, temporary_directory, 1, 1)
+    expect(@file_repository.size).to eq(0)
+    path = ""
+    @file_repository.get_factory(prefix_key) do |factory|
+      factory.current.write("hello")
+      # force a rotation so we get an empty file that will get stale.
+      factory.rotate!
+      path = factory.current.temp_path
+    end
+
+    @file_repository.get_file("another-prefix") { |file| file.write("hello") }
+    expect(@file_repository.size).to eq(2)
+    try(10) { expect(@file_repository.size).to eq(1) }
+    expect(File.directory?(path)).to be_falsey
+  end
+end
+
+
+describe LogStash::Outputs::S3::FileRepository::PrefixedValue do
+  let(:factory) { spy("factory", :current => file) }
+  subject { described_class.new(factory, 1) }
+
+  context "#stale?" do
+    context "the file is empty and older than stale time" do
+      let(:file) { double("file", :size => 0, :ctime => Time.now - 5) }
+
+      it "returns true" do
+        expect(subject.stale?).to be_truthy
+      end
+    end
+
+    context "when the file has data in it" do
+      let(:file) { double("file", :size => 200, :ctime => Time.now - 5) }
+
+      it "returns false" do
+        expect(subject.stale?).to be_falsey
+      end
+    end
+
+    context "when the file is not old enough" do
+      let(:file) { double("file", :size => 0, :ctime => Time.now + 100) }
+
+      it "returns false" do
+        expect(subject.stale?).to be_falsey
+      end
+    end
+  end
+end

data/spec/outputs/s3/size_and_time_rotation_policy_spec.rb

@@ -0,0 +1,77 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/s3/size_and_time_rotation_policy"
+require "logstash/outputs/s3/temporary_file"
+
+describe LogStash::Outputs::S3::SizeAndTimeRotationPolicy do
+  let(:file_size) { 10 }
+  let(:time_file) { 1 }
+  subject { described_class.new(file_size, time_file) }
+
+  let(:temporary_directory) { Stud::Temporary.pathname }
+  let(:temporary_file) { Stud::Temporary.file }
+  let(:name) { "foobar" }
+  let(:content) { "hello" * 1000 }
+  let(:file) { LogStash::Outputs::S3::TemporaryFile.new(name, temporary_file, temporary_directory) }
+
+  it "raises an exception if the `time_file` is set to 0" do
+    expect { described_class.new(100, 0) }.to raise_error(LogStash::ConfigurationError, /time_file/)
+  end
+
+  it "raises an exception if the `time_file` is < 0" do
+    expect { described_class.new(100, -100) }.to raise_error(LogStash::ConfigurationError, /time_file/)
+  end
+
+  it "raises an exception if the `size_file` is 0" do
+    expect { described_class.new(0, 100) }.to raise_error(LogStash::ConfigurationError, /size_file/)
+  end
+
+  it "raises an exception if the `size_file` is < 0" do
+    expect { described_class.new(-100, 100) }.to raise_error(LogStash::ConfigurationError, /size_file/)
+  end
+
+  it "returns true if the size on disk is higher than the `file_size`" do
+    file.write(content)
+    file.fsync
+    expect(subject.rotate?(file)).to be_truthy
+  end
+
+  it "returns false if the size is inferior than the `file_size`" do
+    expect(subject.rotate?(file)).to be_falsey
+  end
+
+  context "when the size of the file is superior to 0" do
+    let(:file_size) { 10000 }
+
+    before :each do
+      file.write(content)
+      file.fsync
+    end
+
+    it "returns true if the file old enough" do
+      allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60) )
+      expect(subject.rotate?(file)).to be_truthy
+    end
+
+    it "returns false is not old enough" do
+      allow(file).to receive(:ctime).and_return(Time.now + time_file * 10)
+      expect(subject.rotate?(file)).to be_falsey
+    end
+  end
+
+  context "When the size of the file is 0" do
+    it "returns false if the file old enough" do
+      expect(subject.rotate?(file)).to be_falsey
+    end
+
+    it "returns false is not old enough" do
+      expect(subject.rotate?(file)).to be_falsey
+    end
+  end
+
+  context "#needs_periodic?" do
+    it "return true" do
+      expect(subject.needs_periodic?).to be_truthy
+    end
+  end
+end

data/spec/outputs/s3/size_rotation_policy_spec.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/s3/size_rotation_policy"
+require "logstash/outputs/s3/temporary_file"
+require "fileutils"
+
+describe LogStash::Outputs::S3::SizeRotationPolicy do
+  subject { described_class.new(size_file) }
+
+  let(:temporary_directory) {  Stud::Temporary.directory }
+  let(:temporary_file) { Stud::Temporary.file }
+  let(:name) { "foobar" }
+  let(:content) { "hello" * 1000 }
+  let(:size_file) { 10 } # in bytes
+  let(:file) { LogStash::Outputs::S3::TemporaryFile.new(name, temporary_file, temporary_directory) }
+
+  it "returns true if the size on disk is higher than the `size_file`" do
+    file.write(content)
+    file.fsync
+    expect(subject.rotate?(file)).to be_truthy
+  end
+
+  it "returns false if the size is inferior than the `size_file`" do
+    expect(subject.rotate?(file)).to be_falsey
+  end
+
+  it "raises an exception if the `size_file` is 0" do
+    expect { described_class.new(0) }.to raise_error(LogStash::ConfigurationError, /need to be greather than 0/)
+  end
+
+  it "raises an exception if the `size_file` is < 0" do
+    expect { described_class.new(-100) }.to raise_error(LogStash::ConfigurationError, /need to be greather than 0/)
+  end
+
+  context "#needs_periodic?" do
+    it "return false" do
+      expect(subject.needs_periodic?).to be_falsey
+    end
+  end
+
+end

data/spec/outputs/s3/temporary_file_factory_spec.rb

@@ -0,0 +1,89 @@
+# encoding: utf-8
+require "logstash/outputs/s3/temporary_file_factory"
+require "logstash/outputs/s3/temporary_file"
+require "stud/temporary"
+require "fileutils"
+
+describe LogStash::Outputs::S3::TemporaryFileFactory do
+  let(:prefix) { "foobar" }
+  let(:tags) { [] }
+  let(:temporary_directory) { Stud::Temporary.pathname }
+
+  before do
+    FileUtils.mkdir_p(temporary_directory)
+  end
+
+  subject { described_class.new(prefix, tags, encoding, temporary_directory) }
+
+  shared_examples "file factory" do
+    it "creates the file on disk" do
+      expect(File.exist?(subject.current.path)).to be_truthy
+    end
+
+    it "returns a size equal to zero after file creation" do
+      expect(subject.current.size).to eq(0)
+    end
+
+    it "create a temporary file when initialized" do
+      expect(subject.current).to be_kind_of(LogStash::Outputs::S3::TemporaryFile)
+    end
+
+    it "create a file in the right format" do
+      expect(subject.current.path).to match(extension)
+    end
+
+    it "allow to rotate the file" do
+      file_path = subject.current.path
+      expect(subject.rotate!.path).not_to eq(file_path)
+    end
+
+    it "increments the part name on rotation" do
+      expect(subject.current.path).to match(/part0/)
+      expect(subject.rotate!.path).to match(/part1/)
+    end
+
+    it "includes the date" do
+      n = Time.now
+      expect(subject.current.path).to include(n.strftime("%Y-%m-%dT"))
+    end
+
+    it "include the file key in the path"  do
+      file = subject.current
+      expect(file.path).to match(/#{file.key}/)
+    end
+
+    it "create a unique directory in the temporary directory for each file" do
+      uuid = "hola"
+      expect(SecureRandom).to receive(:uuid).and_return(uuid).twice
+      expect(subject.current.path).to include(uuid)
+    end
+
+    context "with tags supplied" do
+      let(:tags) { ["secret", "service"] }
+
+      it "adds tags to the filename" do
+        expect(subject.current.path).to match(/tag_#{tags.join('.')}.part/)
+      end
+    end
+
+    context "without tags" do
+      it "doesn't add tags to the filename" do
+        expect(subject.current.path).not_to match(/tag_/)
+      end
+    end
+  end
+
+  context "when gzip" do
+    let(:encoding) { "gzip" }
+    let(:extension) { /\.txt.gz$/ }
+
+    include_examples "file factory"
+  end
+
+  context "when encoding set to `none`" do
+    let(:encoding) { "none" }
+    let(:extension) { /\.txt$/ }
+
+    include_examples "file factory"
+  end
+end

data/spec/outputs/s3/temporary_file_spec.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/s3/temporary_file"
+require "stud/temporary"
+require "fileutils"
+require "securerandom"
+
+describe LogStash::Outputs::S3::TemporaryFile do
+  let(:content) { "hello world" }
+  let(:key) { "foo" }
+  let(:uuid) { SecureRandom.uuid }
+  let(:temporary_file) { ::File.open(::File.join(temporary_directory, uuid, key), "w+") }
+  let(:temporary_directory) {  Stud::Temporary.directory }
+
+  before :each do
+    FileUtils.mkdir_p(::File.join(temporary_directory, uuid))
+  end
+
+  subject { described_class.new(key, temporary_file, temporary_directory) }
+
+  it "returns the key of the file" do
+    expect(subject.key).to eq(key)
+  end
+
+  it "saves content to a file" do
+    subject.write(content)
+    subject.close
+    expect(File.read(subject.path).strip).to eq(content)
+  end
+
+  it "deletes a file" do
+    expect(File.exist?(subject.path)).to be_truthy
+    subject.delete!
+    expect(File.exist?(subject.path)).to be_falsey
+  end
+
+  it "successfully delete a file already closed" do
+    subject.close
+    expect(File.exist?(subject.path)).to be_truthy
+    subject.delete!
+    expect(File.exist?(subject.path)).to be_falsey
+  end
+
+  it "returns the creation time" do
+    expect(subject.ctime).to be < Time.now + 0.5
+  end
+end

data/spec/outputs/s3/time_rotation_policy_spec.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/s3/time_rotation_policy"
+require "logstash/outputs/s3/temporary_file"
+
+describe LogStash::Outputs::S3::TimeRotationPolicy do
+  subject { described_class.new(max_time) }
+
+  let(:max_time) { 1 }
+  let(:temporary_directory) {  Stud::Temporary.directory }
+  let(:temporary_file) { Stud::Temporary.file }
+  let(:name) { "foobar" }
+  let(:content) { "hello" * 1000 }
+  let(:file) { LogStash::Outputs::S3::TemporaryFile.new(name, temporary_file, temporary_directory) }
+
+  it "raises an exception if the `file_time` is set to 0" do
+    expect { described_class.new(0) }.to raise_error(LogStash::ConfigurationError, /`time_file` need to be greather than 0/)
+  end
+
+  it "raises an exception if the `file_time` is < 0" do
+    expect { described_class.new(-100) }.to raise_error(LogStash::ConfigurationError, /`time_file` need to be greather than 0/)
+  end
+
+  context "when the size of the file is superior to 0" do
+    before :each do
+      file.write(content)
+      file.fsync
+    end
+
+    it "returns true if the file old enough" do
+      allow(file).to receive(:ctime).and_return(Time.now - (max_time * 2 * 60))
+      expect(subject.rotate?(file)).to be_truthy
+    end
+
+    it "returns false is not old enough" do
+      expect(subject.rotate?(file)).to be_falsey
+    end
+  end
+
+  context "When the size of the file is 0" do
+    it "returns false if the file old enough" do
+      allow(file).to receive(:ctime).and_return(Time.now - (max_time * 2 * 60))
+      expect(subject.rotate?(file)).to be_falsey
+    end
+
+    it "returns false is not old enough" do
+      expect(subject.rotate?(file)).to be_falsey
+    end
+  end
+
+  context "#needs_periodic?" do
+    it "return false" do
+      expect(subject.needs_periodic?).to be_truthy
+    end
+  end
+
+  it "convert minute into seconds" do
+    expect(subject.time_file).to eq(60)
+  end
+end

data/spec/outputs/s3/uploader_spec.rb

@@ -0,0 +1,69 @@
+# Encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/s3/uploader"
+require "logstash/outputs/s3/temporary_file"
+require "stud/temporary"
+
+describe LogStash::Outputs::S3::Uploader do
+  let(:logger) { spy(:logger) }
+  let(:max_upload_workers) { 1 }
+  let(:bucket_name) { "foobar-bucket" }
+  let(:client) { Aws::S3::Client.new(stub_responses: true) }
+  let(:bucket) { Aws::S3::Bucket.new(bucket_name, :client => client) }
+  let(:temporary_directory) { Stud::Temporary.pathname }
+  let(:temporary_file) { Stud::Temporary.file }
+  let(:key) { "foobar" }
+  let(:threadpool) do
+    Concurrent::ThreadPoolExecutor.new({
+                                         :min_threads => 1,
+                                         :max_threads => 8,
+                                         :max_queue => 1,
+                                         :fallback_policy => :caller_runs
+                                       })
+  end
+
+  let(:file) do
+    f = LogStash::Outputs::S3::TemporaryFile.new(key, temporary_file, temporary_directory)
+    f.write("random content")
+    f.fsync
+    f
+  end
+
+  subject { described_class.new(bucket, logger, threadpool, retry_delay: 0.01) }
+
+  it "upload file to the s3 bucket" do
+    expect { subject.upload(file) }.not_to raise_error
+  end
+
+  it "execute a callback when the upload is complete" do
+    callback = proc { |_| }
+
+    expect(callback).to receive(:call).with(file)
+    subject.upload(file, :on_complete => callback)
+  end
+
+  it "retries errors indefinitely" do
+    s3 = double("s3").as_null_object
+
+    allow(bucket).to receive(:object).with(file.key).and_return(s3)
+
+    expect(logger).to receive(:warn).with(any_args)
+    expect(s3).to receive(:upload_file).with(any_args).and_raise(RuntimeError.new('UPLOAD FAILED')).exactly(5).times
+    expect(s3).to receive(:upload_file).with(any_args).and_return(true)
+
+    subject.upload(file)
+  end
+
+  it "retries errors specified times" do
+    subject = described_class.new(bucket, logger, threadpool, retry_count: 3, retry_delay: 0.01)
+    s3 = double("s3").as_null_object
+
+    allow(bucket).to receive(:object).with(file.key).and_return(s3)
+
+    expect(logger).to receive(:warn).with(any_args).exactly(3).times
+    expect(logger).to receive(:error).with(any_args).once
+    expect(s3).to receive(:upload_file).with(file.path, {}).and_raise(RuntimeError).at_least(1).times
+
+    subject.upload(file)
+  end
+end

data/spec/outputs/s3/writable_directory_validator_spec.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/s3/writable_directory_validator"
+require "stud/temporary"
+
+describe LogStash::Outputs::S3::WritableDirectoryValidator do
+  let(:temporary_directory) {  File.join(Stud::Temporary.directory, Time.now.to_i.to_s) }
+
+  subject { described_class }
+
+  context "when the directory doesn't exists" do
+    it "creates the directory" do
+      expect(Dir.exist?(temporary_directory)).to be_falsey
+      expect(subject.valid?(temporary_directory)).to be_truthy
+      expect(Dir.exist?(temporary_directory)).to be_truthy
+    end
+  end
+
+  context "when the directory exist" do
+    before do
+      FileUtils.mkdir_p(temporary_directory)
+    end
+
+    it "doesn't change the directory" do
+      expect(Dir.exist?(temporary_directory)).to be_truthy
+      expect(subject.valid?(temporary_directory)).to be_truthy
+      expect(Dir.exist?(temporary_directory)).to be_truthy
+    end
+  end
+
+  it "return false if the directory is not writable" do
+    expect(::File).to receive(:writable?).with(temporary_directory).and_return(false)
+    expect(subject.valid?(temporary_directory)).to be_falsey
+  end
+
+  it "return true if the directory is writable" do
+    expect(::File).to receive(:writable?).with(temporary_directory).and_return(true)
+    expect(subject.valid?(temporary_directory)).to be_truthy
+  end
+end

data/spec/outputs/s3/write_bucket_permission_validator_spec.rb

@@ -0,0 +1,49 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/s3/write_bucket_permission_validator"
+
+describe LogStash::Outputs::S3::WriteBucketPermissionValidator do
+  let(:logger) { spy(:logger ) }
+  let(:bucket_name) { "foobar" }
+  let(:obj) { double("s3_object") }
+  let(:client) { Aws::S3::Client.new(stub_responses: true) }
+  let(:bucket) { Aws::S3::Bucket.new(bucket_name, :client => client) }
+  let(:upload_options) { {} }
+
+  subject { described_class.new(logger) }
+
+  before do
+    expect(bucket).to receive(:object).with(any_args).and_return(obj)
+  end
+
+  context 'when using upload_options' do
+    let(:upload_options) {{ :server_side_encryption => true }}
+    it 'they are passed through to upload_file' do
+      expect(obj).to receive(:upload_file).with(anything, upload_options)
+      expect(obj).to receive(:delete).and_return(true)
+      expect(subject.valid?(bucket, upload_options)).to be_truthy
+    end
+
+  end
+
+  context "when permissions are sufficient" do
+    it "returns true" do
+      expect(obj).to receive(:upload_file).with(any_args).and_return(true)
+      expect(obj).to receive(:delete).and_return(true)
+      expect(subject.valid?(bucket, upload_options)).to be_truthy
+    end
+
+    it "hides delete errors" do
+      expect(obj).to receive(:upload_file).with(any_args).and_return(true)
+      expect(obj).to receive(:delete).and_raise(StandardError)
+      expect(subject.valid?(bucket, upload_options)).to be_truthy
+    end
+  end
+
+  context "when permission aren't sufficient" do
+    it "returns false" do
+      expect(obj).to receive(:upload_file).with(any_args).and_raise(StandardError)
+      expect(subject.valid?(bucket, upload_options)).to be_falsey
+    end
+  end
+end