logstash-output-azure 0.1.0

data/spec/outputs/blob/file_repository_spec.rb

@@ -0,0 +1,143 @@
+# encoding: utf-8
+require "logstash/outputs/Logstash_Azure_Blob_Output"
+require "stud/temporary"
+require "fileutils"
+require_relative "../../spec_helper"
+
+describe LogStash::Outputs::LogstashAzureBlobOutput::FileRepository do
+  let(:tags) { [] }
+  let(:encoding) { "none" }
+  let(:temporary_directory) { Stud::Temporary.pathname }
+  let(:prefix_key) { "a-key" }
+
+  before do
+    FileUtils.mkdir_p(temporary_directory)
+  end
+
+  subject { described_class.new(tags, encoding, temporary_directory) }
+
+  it "returns a temporary file" do
+    subject.get_file(prefix_key) do |file|
+      expect(file).to be_kind_of(LogStash::Outputs::LogstashAzureBlobOutput::TemporaryFile)
+    end
+  end
+
+  it "returns the same file for the same prefix key" do
+    file_path = nil
+
+    subject.get_file(prefix_key) do |file|
+      file_path = file.path
+    end
+
+    subject.get_file(prefix_key) do |file|
+      expect(file.path).to eq(file_path)
+    end
+  end
+
+  it "returns the same file for the same dynamic prefix key" do
+    prefix = "%{type}/%{+YYYY}/%{+MM}/%{+dd}/"
+    event = LogStash::Event.new({ "type" => "syslog"})
+    key = event.sprintf(prefix)
+    file_path = nil
+
+
+    subject.get_file(key) do |file|
+      file_path = file.path
+    end
+
+    subject.get_file(key) do |file|
+      expect(file.path).to eq(file_path)
+    end
+  end
+
+  it "returns different file for different prefix keys" do
+    file_path = nil
+
+    subject.get_file(prefix_key) do |file|
+      file_path = file.path
+    end
+
+    subject.get_file("another_prefix_key") do |file|
+      expect(file.path).not_to eq(file_path)
+    end
+  end
+
+  it "allows to get the file factory for a specific prefix" do
+    subject.get_factory(prefix_key) do |factory|
+      expect(factory).to be_kind_of(LogStash::Outputs::LogstashAzureBlobOutput::TemporaryFileFactory)
+    end
+  end
+
+  it "returns a different file factory for a different prefix keys" do
+    factory = nil
+
+    subject.get_factory(prefix_key) do |f|
+      factory = f
+    end
+
+    subject.get_factory("another_prefix_key") do |f|
+      expect(factory).not_to eq(f)
+    end
+  end
+
+  it "returns the number of prefix keys" do
+    expect(subject.size).to eq(0)
+    subject.get_file(prefix_key) { |file| file.write("something") }
+    expect(subject.size).to eq(1)
+  end
+
+  it "returns all available keys" do
+    subject.get_file(prefix_key) { |file| file.write("something") }
+    expect(subject.keys.toArray).to include(prefix_key)
+    expect(subject.keys.toArray.size).to eq(1)
+  end
+
+  it "clean stale factories" do
+    @file_repository = described_class.new(tags, encoding, temporary_directory, 1, 1)
+    expect(@file_repository.size).to eq(0)
+    path = ""
+    @file_repository.get_factory(prefix_key) do |factory|
+      factory.current.write("hello")
+      # force a rotation so we get an empty file that will get stale.
+      factory.rotate!
+      path = factory.current.temp_path
+    end
+
+    @file_repository.get_file("another-prefix") { |file| file.write("hello") }
+    expect(@file_repository.size).to eq(2)
+    try(10) { expect(@file_repository.size).to eq(1) }
+    expect(File.directory?(path)).to be_falsey
+  end
+end
+
+
+describe LogStash::Outputs::LogstashAzureBlobOutput::FileRepository::PrefixedValue do
+  let(:factory) { spy("factory", :current => file) }
+  subject { described_class.new(factory, 1) }
+
+  context "#stale?" do
+    context "the file is empty and older than stale time" do
+      let(:file) { double("file", :size => 0, :ctime => Time.now - 5) }
+
+      it "returns true" do
+        expect(subject.stale?).to be_truthy
+      end
+    end
+
+    context "when the file has data in it" do
+      let(:file) { double("file", :size => 200, :ctime => Time.now - 5) }
+
+      it "returns false" do
+        expect(subject.stale?).to be_falsey
+      end
+    end
+
+    context "when the file is not old enough" do
+      let(:file) { double("file", :size => 0, :ctime => Time.now + 100) }
+
+      it "returns false" do
+        expect(subject.stale?).to be_falsey
+      end
+    end
+  end
+end

data/spec/outputs/blob/size_and_time_rotation_policy_spec.rb

@@ -0,0 +1,77 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/blob/size_and_time_rotation_policy"
+require "logstash/outputs/blob/temporary_file"
+
+describe LogStash::Outputs::LogstashAzureBlobOutput::SizeAndTimeRotationPolicy do
+  let(:file_size) { 10 }
+  let(:time_file) { 1 }
+  subject { described_class.new(file_size, time_file) }
+
+  let(:temporary_directory) { Stud::Temporary.pathname }
+  let(:temporary_file) { Stud::Temporary.file }
+  let(:name) { "foobar" }
+  let(:content) { "hello" * 1000 }
+  let(:file) { LogStash::Outputs::LogstashAzureBlobOutput::TemporaryFile.new(name, temporary_file, temporary_directory) }
+
+  it "raises an exception if the `time_file` is set to 0" do
+    expect { described_class.new(100, 0) }.to raise_error(LogStash::ConfigurationError, /time_file/)
+  end
+
+  it "raises an exception if the `time_file` is < 0" do
+    expect { described_class.new(100, -100) }.to raise_error(LogStash::ConfigurationError, /time_file/)
+  end
+
+  it "raises an exception if the `size_file` is 0" do
+    expect { described_class.new(0, 100) }.to raise_error(LogStash::ConfigurationError, /size_file/)
+  end
+
+  it "raises an exception if the `size_file` is < 0" do
+    expect { described_class.new(-100, 100) }.to raise_error(LogStash::ConfigurationError, /size_file/)
+  end
+
+  it "returns true if the size on disk is higher than the `file_size`" do
+    file.write(content)
+    file.fsync
+    expect(subject.rotate?(file)).to be_truthy
+  end
+
+  it "returns false if the size is inferior than the `file_size`" do
+    expect(subject.rotate?(file)).to be_falsey
+  end
+
+  context "when the size of the file is superior to 0" do
+    let(:file_size) { 10000 }
+
+    before :each do
+      file.write(content)
+      file.fsync
+    end
+
+    it "returns true if the file old enough" do
+      allow(file).to receive(:ctime).and_return(Time.now - (time_file * 2 * 60) )
+      expect(subject.rotate?(file)).to be_truthy
+    end
+
+    it "returns false is not old enough" do
+      allow(file).to receive(:ctime).and_return(Time.now + time_file * 10)
+      expect(subject.rotate?(file)).to be_falsey
+    end
+  end
+
+  context "When the size of the file is 0" do
+    it "returns false if the file old enough" do
+      expect(subject.rotate?(file)).to be_falsey
+    end
+
+    it "returns false is not old enough" do
+      expect(subject.rotate?(file)).to be_falsey
+    end
+  end
+
+  context "#needs_periodic?" do
+    it "return true" do
+      expect(subject.needs_periodic?).to be_truthy
+    end
+  end
+end

data/spec/outputs/blob/size_rotation_policy_spec.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/blob/size_rotation_policy"
+require "logstash/outputs/blob/temporary_file"
+require "fileutils"
+
+describe LogStash::Outputs::LogstashAzureBlobOutput::SizeRotationPolicy do
+  subject { described_class.new(size_file) }
+
+  let(:temporary_directory) {  Stud::Temporary.directory }
+  let(:temporary_file) { Stud::Temporary.file }
+  let(:name) { "foobar" }
+  let(:content) { "hello" * 1000 }
+  let(:size_file) { 10 } # in bytes
+  let(:file) { LogStash::Outputs::LogstashAzureBlobOutput::TemporaryFile.new(name, temporary_file, temporary_directory) }
+
+  it "returns true if the size on disk is higher than the `size_file`" do
+    file.write(content)
+    file.fsync
+    expect(subject.rotate?(file)).to be_truthy
+  end
+
+  it "returns false if the size is inferior than the `size_file`" do
+    expect(subject.rotate?(file)).to be_falsey
+  end
+
+  it "raises an exception if the `size_file` is 0" do
+    expect { described_class.new(0) }.to raise_error(LogStash::ConfigurationError, /need to be greather than 0/)
+  end
+
+  it "raises an exception if the `size_file` is < 0" do
+    expect { described_class.new(-100) }.to raise_error(LogStash::ConfigurationError, /need to be greather than 0/)
+  end
+
+  context "#needs_periodic?" do
+    it "return false" do
+      expect(subject.needs_periodic?).to be_falsey
+    end
+  end
+
+end

data/spec/outputs/blob/temporary_file_factory_spec.rb

@@ -0,0 +1,89 @@
+# encoding: utf-8
+require "logstash/outputs/blob/temporary_file_factory"
+require "logstash/outputs/blob/temporary_file"
+require "stud/temporary"
+require "fileutils"
+
+describe LogStash::Outputs::LogstashAzureBlobOutput::TemporaryFileFactory do
+  let(:prefix) { "foobar" }
+  let(:tags) { [] }
+  let(:temporary_directory) { Stud::Temporary.pathname }
+
+  before do
+    FileUtils.mkdir_p(temporary_directory)
+  end
+
+  subject { described_class.new(prefix, tags, encoding, temporary_directory) }
+
+  shared_examples "file factory" do
+    it "creates the file on disk" do
+      expect(File.exist?(subject.current.path)).to be_truthy
+    end
+
+    it "returns a size equal to zero after file creation" do
+      expect(subject.current.size).to eq(0)
+    end
+
+    it "create a temporary file when initialized" do
+      expect(subject.current).to be_kind_of(LogStash::Outputs::LogstashAzureBlobOutput::TemporaryFile)
+    end
+
+    it "create a file in the right format" do
+      expect(subject.current.path).to match(extension)
+    end
+
+    it "allow to rotate the file" do
+      file_path = subject.current.path
+      expect(subject.rotate!.path).not_to eq(file_path)
+    end
+
+    it "increments the part name on rotation" do
+      expect(subject.current.path).to match(/part0/)
+      expect(subject.rotate!.path).to match(/part1/)
+    end
+
+    it "includes the date" do
+      n = Time.now
+      expect(subject.current.path).to include(n.strftime("%Y-%m-%dT"))
+    end
+
+    it "include the file key in the path"  do
+      file = subject.current
+      expect(file.path).to match(/#{file.key}/)
+    end
+
+    it "create a unique directory in the temporary directory for each file" do
+      uuid = "hola"
+      expect(SecureRandom).to receive(:uuid).and_return(uuid).twice
+      expect(subject.current.path).to include(uuid)
+    end
+
+    context "with tags supplied" do
+      let(:tags) { ["secret", "service"] }
+
+      it "adds tags to the filename" do
+        expect(subject.current.path).to match(/tag_#{tags.join('.')}.part/)
+      end
+    end
+
+    context "without tags" do
+      it "doesn't add tags to the filename" do
+        expect(subject.current.path).not_to match(/tag_/)
+      end
+    end
+  end
+
+  context "when gzip" do
+    let(:encoding) { "gzip" }
+    let(:extension) { /\.txt.gz$/ }
+
+    include_examples "file factory"
+  end
+
+  context "when encoding set to `none`" do
+    let(:encoding) { "none" }
+    let(:extension) { /\.txt$/ }
+
+    include_examples "file factory"
+  end
+end

data/spec/outputs/blob/temporary_file_spec.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/blob/temporary_file"
+require "stud/temporary"
+require "fileutils"
+require "securerandom"
+
+describe LogStash::Outputs::LogstashAzureBlobOutput::TemporaryFile do
+  let(:content) { "hello world" }
+  let(:key) { "foo" }
+  let(:uuid) { SecureRandom.uuid }
+  let(:temporary_file) { ::File.open(::File.join(temporary_directory, uuid, key), "w+") }
+  let(:temporary_directory) {  Stud::Temporary.directory }
+
+  before :each do
+    FileUtils.mkdir_p(::File.join(temporary_directory, uuid))
+  end
+
+  subject { described_class.new(key, temporary_file, temporary_directory) }
+
+  it "returns the key of the file" do
+    expect(subject.key).to eq(key)
+  end
+
+  it "saves content to a file" do
+    subject.write(content)
+    subject.close
+    expect(File.read(subject.path).strip).to eq(content)
+  end
+
+  it "deletes a file" do
+    expect(File.exist?(subject.path)).to be_truthy
+    subject.delete!
+    expect(File.exist?(subject.path)).to be_falsey
+  end
+
+  it "successfully delete a file already closed" do
+    subject.close
+    expect(File.exist?(subject.path)).to be_truthy
+    subject.delete!
+    expect(File.exist?(subject.path)).to be_falsey
+  end
+
+  it "returns the creation time" do
+    expect(subject.ctime).to be < Time.now + 0.5
+  end
+end

data/spec/outputs/blob/time_rotation_policy_spec.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/blob/time_rotation_policy"
+require "logstash/outputs/blob/temporary_file"
+
+describe LogStash::Outputs::LogstashAzureBlobOutput::TimeRotationPolicy do
+  subject { described_class.new(max_time) }
+
+  let(:max_time) { 1 }
+  let(:temporary_directory) {  Stud::Temporary.directory }
+  let(:temporary_file) { Stud::Temporary.file }
+  let(:name) { "foobar" }
+  let(:content) { "hello" * 1000 }
+  let(:file) { LogStash::Outputs::LogstashAzureBlobOutput::TemporaryFile.new(name, temporary_file, temporary_directory) }
+
+  it "raises an exception if the `file_time` is set to 0" do
+    expect { described_class.new(0) }.to raise_error(LogStash::ConfigurationError, /`time_file` need to be greather than 0/)
+  end
+
+  it "raises an exception if the `file_time` is < 0" do
+    expect { described_class.new(-100) }.to raise_error(LogStash::ConfigurationError, /`time_file` need to be greather than 0/)
+  end
+
+  context "when the size of the file is superior to 0" do
+    before :each do
+      file.write(content)
+      file.fsync
+    end
+
+    it "returns true if the file old enough" do
+      allow(file).to receive(:ctime).and_return(Time.now - (max_time * 2 * 60))
+      expect(subject.rotate?(file)).to be_truthy
+    end
+
+    it "returns false is not old enough" do
+      expect(subject.rotate?(file)).to be_falsey
+    end
+  end
+
+  context "When the size of the file is 0" do
+    it "returns false if the file old enough" do
+      allow(file).to receive(:ctime).and_return(Time.now - (max_time * 2 * 60))
+      expect(subject.rotate?(file)).to be_falsey
+    end
+
+    it "returns false is not old enough" do
+      expect(subject.rotate?(file)).to be_falsey
+    end
+  end
+
+  context "#needs_periodic?" do
+    it "return false" do
+      expect(subject.needs_periodic?).to be_truthy
+    end
+  end
+
+  it "convert minute into seconds" do
+    expect(subject.time_file).to eq(60)
+  end
+end